OP#792 initial commit after fork

cmd/healthcheck-handler.go

@@ -0,0 +1,212 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"net/http"
+	"strconv"
+	"time"
+
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
+)
+
+const unavailable = "offline"
+
+func checkHealth(w http.ResponseWriter) ObjectLayer {
+	objLayer := newObjectLayerFn()
+	if objLayer == nil {
+		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
+		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
+		return nil
+	}
+
+	if !globalBucketMetadataSys.Initialized() {
+		w.Header().Set(xhttp.MinIOServerStatus, "bucket-metadata-offline")
+		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
+		return nil
+	}
+
+	if !globalIAMSys.Initialized() {
+		w.Header().Set(xhttp.MinIOServerStatus, "iam-offline")
+		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
+		return nil
+	}
+
+	return objLayer
+}
+
+// ClusterCheckHandler returns if the server is ready for requests.
+func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ClusterCheckHandler")
+
+	objLayer := checkHealth(w)
+	if objLayer == nil {
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, globalAPIConfig.getClusterDeadline())
+	defer cancel()
+
+	opts := HealthOptions{
+		Maintenance:    r.Form.Get("maintenance") == "true",
+		DeploymentType: r.Form.Get("deployment-type"),
+	}
+	result := objLayer.Health(ctx, opts)
+	w.Header().Set(xhttp.MinIOWriteQuorum, strconv.Itoa(result.WriteQuorum))
+	w.Header().Set(xhttp.MinIOStorageClassDefaults, strconv.FormatBool(result.UsingDefaults))
+	// return how many drives are being healed if any
+	if result.HealingDrives > 0 {
+		w.Header().Set(xhttp.MinIOHealingDrives, strconv.Itoa(result.HealingDrives))
+	}
+	if !result.Healthy {
+		// As a maintenance call we are purposefully asked to be taken
+		// down, this is for orchestrators to know if we can safely
+		// take this server down, return appropriate error.
+		if opts.Maintenance {
+			writeResponse(w, http.StatusPreconditionFailed, nil, mimeNone)
+		} else {
+			writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
+		}
+		return
+	}
+	writeResponse(w, http.StatusOK, nil, mimeNone)
+}
+
+// ClusterReadCheckHandler returns if the server is ready for requests.
+func ClusterReadCheckHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ClusterReadCheckHandler")
+
+	objLayer := checkHealth(w)
+	if objLayer == nil {
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, globalAPIConfig.getClusterDeadline())
+	defer cancel()
+
+	opts := HealthOptions{
+		Maintenance:    r.Form.Get("maintenance") == "true",
+		DeploymentType: r.Form.Get("deployment-type"),
+	}
+	result := objLayer.Health(ctx, opts)
+	w.Header().Set(xhttp.MinIOReadQuorum, strconv.Itoa(result.ReadQuorum))
+	w.Header().Set(xhttp.MinIOStorageClassDefaults, strconv.FormatBool(result.UsingDefaults))
+	// return how many drives are being healed if any
+	if result.HealingDrives > 0 {
+		w.Header().Set(xhttp.MinIOHealingDrives, strconv.Itoa(result.HealingDrives))
+	}
+	if !result.HealthyRead {
+		// As a maintenance call we are purposefully asked to be taken
+		// down, this is for orchestrators to know if we can safely
+		// take this server down, return appropriate error.
+		if opts.Maintenance {
+			writeResponse(w, http.StatusPreconditionFailed, nil, mimeNone)
+		} else {
+			writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
+		}
+		return
+	}
+	writeResponse(w, http.StatusOK, nil, mimeNone)
+}
+
+// ReadinessCheckHandler checks whether MinIO is up and ready to serve requests.
+// It also checks whether the KMS is available and whether etcd is reachable,
+// if configured.
+func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
+	if objLayer := newObjectLayerFn(); objLayer == nil {
+		w.Header().Set(xhttp.MinIOServerStatus, unavailable) // Service not initialized yet
+	}
+	if r.Header.Get(xhttp.MinIOPeerCall) != "" {
+		writeResponse(w, http.StatusOK, nil, mimeNone)
+		return
+	}
+
+	if int(globalHTTPStats.loadRequestsInQueue()) > globalAPIConfig.getRequestsPoolCapacity() {
+		apiErr := getAPIError(ErrBusy)
+		switch r.Method {
+		case http.MethodHead:
+			writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone)
+		case http.MethodGet:
+			writeErrorResponse(r.Context(), w, apiErr, r.URL)
+		}
+		return
+	}
+
+	// Verify if KMS is reachable if its configured
+	if GlobalKMS != nil {
+		ctx, cancel := context.WithTimeout(r.Context(), time.Minute)
+		defer cancel()
+
+		if _, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{AssociatedData: kms.Context{"healthcheck": ""}}); err != nil {
+			switch r.Method {
+			case http.MethodHead:
+				apiErr := toAPIError(r.Context(), err)
+				writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone)
+			case http.MethodGet:
+				writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL)
+			}
+			return
+		}
+	}
+
+	if globalEtcdClient != nil {
+		// Borrowed from
+		// https://github.com/etcd-io/etcd/blob/main/etcdctl/ctlv3/command/ep_command.go#L118
+		ctx, cancel := context.WithTimeout(r.Context(), defaultContextTimeout)
+		defer cancel()
+		if _, err := globalEtcdClient.Get(ctx, "health"); err != nil {
+			// etcd unreachable throw an error..
+			switch r.Method {
+			case http.MethodHead:
+				apiErr := toAPIError(r.Context(), err)
+				writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone)
+			case http.MethodGet:
+				writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL)
+			}
+			return
+		}
+	}
+	writeResponse(w, http.StatusOK, nil, mimeNone)
+}
+
+// LivenessCheckHandler checks whether MinIO is up. It differs from the
+// readiness handler since a failing liveness check causes pod restarts
+// in K8S environments. Therefore, it does not contact external systems.
+func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
+	if objLayer := newObjectLayerFn(); objLayer == nil {
+		w.Header().Set(xhttp.MinIOServerStatus, unavailable) // Service not initialized yet
+	}
+	if r.Header.Get(xhttp.MinIOPeerCall) != "" {
+		writeResponse(w, http.StatusOK, nil, mimeNone)
+		return
+	}
+
+	if int(globalHTTPStats.loadRequestsInQueue()) > globalAPIConfig.getRequestsPoolCapacity() {
+		apiErr := getAPIError(ErrBusy)
+		switch r.Method {
+		case http.MethodHead:
+			writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone)
+		case http.MethodGet:
+			writeErrorResponse(r.Context(), w, apiErr, r.URL)
+		}
+		return
+	}
+	writeResponse(w, http.StatusOK, nil, mimeNone)
+}