Implement list, clear locks REST API w/ pkg/madmin support (#3491)

* Filter lock info based on bucket, prefix and time since lock was held
* Implement list and clear locks REST API
* madmin: Add list and clear locks API
* locks: Clear locks matching bucket, prefix, relTime.
* Gather lock information across nodes for both list and clear locks admin REST API.
* docs: Add lock API to management APIs

cmd/admin-handlers.go

@@ -19,6 +19,8 @@ package cmd
 import (
 	"encoding/json"
 	"net/http"
+	"net/url"
+	"time"
 )
 
 const (
@@ -28,9 +30,8 @@ const (
 // ServiceStatusHandler - GET /?service
 // HTTP header x-minio-operation: status
 // ----------
-// This implementation of the GET operation fetches server status information.
-// provides total disk space available to use, online disks, offline disks and
-// quorum threshold.
+// Fetches server status information like total disk space available
+// to use, online disks, offline disks and quorum threshold.
 func (adminAPI adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Request) {
 	adminAPIErr := checkRequestAuthType(r, "", "", "")
 	if adminAPIErr != ErrNone {
@@ -44,15 +45,16 @@ func (adminAPI adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *
 		errorIf(err, "Failed to marshal storage info into json.")
 		return
 	}
+	// Reply with storage information (across nodes in a
+	// distributed setup) as json.
 	writeSuccessResponse(w, jsonBytes)
 }
 
 // ServiceStopHandler - POST /?service
 // HTTP header x-minio-operation: stop
 // ----------
-// This implementation of the POST operation stops minio server gracefully,
-// in a distributed setup stops all the servers in the cluster. Body sent
-// if any on client request is ignored.
+// Stops minio server gracefully. In a distributed setup, stops all the
+// servers in the cluster.
 func (adminAPI adminAPIHandlers) ServiceStopHandler(w http.ResponseWriter, r *http.Request) {
 	adminAPIErr := checkRequestAuthType(r, "", "", "")
 	if adminAPIErr != ErrNone {
@@ -67,9 +69,8 @@ func (adminAPI adminAPIHandlers) ServiceStopHandler(w http.ResponseWriter, r *ht
 // ServiceRestartHandler - POST /?service
 // HTTP header x-minio-operation: restart
 // ----------
-// This implementation of the POST operation restarts minio server gracefully,
-// in a distributed setup restarts all the servers in the cluster. Body sent
-// if any on client request is ignored.
+// Restarts minio server gracefully. In a distributed setup,  restarts
+// all the servers in the cluster.
 func (adminAPI adminAPIHandlers) ServiceRestartHandler(w http.ResponseWriter, r *http.Request) {
 	adminAPIErr := checkRequestAuthType(r, "", "", "")
 	if adminAPIErr != ErrNone {
@@ -80,3 +81,129 @@ func (adminAPI adminAPIHandlers) ServiceRestartHandler(w http.ResponseWriter, r
 	w.WriteHeader(http.StatusOK)
 	sendServiceCmd(globalAdminPeers, serviceRestart)
 }
+
+// Type-safe lock query params.
+type lockQueryKey string
+
+// Only valid query params for list/clear locks management APIs.
+const (
+	lockBucket    lockQueryKey = "bucket"
+	lockPrefix    lockQueryKey = "prefix"
+	lockOlderThan lockQueryKey = "older-than"
+)
+
+// validateLockQueryParams - Validates query params for list/clear locks management APIs.
+func validateLockQueryParams(vars url.Values) (string, string, time.Duration, APIErrorCode) {
+	bucket := vars.Get(string(lockBucket))
+	prefix := vars.Get(string(lockPrefix))
+	relTimeStr := vars.Get(string(lockOlderThan))
+
+	// N B empty bucket name is invalid
+	if !IsValidBucketName(bucket) {
+		return "", "", time.Duration(0), ErrInvalidBucketName
+	}
+
+	// empty prefix is valid.
+	if !IsValidObjectPrefix(prefix) {
+		return "", "", time.Duration(0), ErrInvalidObjectName
+	}
+
+	// If older-than parameter was empty then set it to 0s to list
+	// all locks older than now.
+	if relTimeStr == "" {
+		relTimeStr = "0s"
+	}
+	relTime, err := time.ParseDuration(relTimeStr)
+	if err != nil {
+		errorIf(err, "Failed to parse duration passed as query value.")
+		return "", "", time.Duration(0), ErrInvalidDuration
+	}
+
+	return bucket, prefix, relTime, ErrNone
+}
+
+// ListLocksHandler - GET /?lock&bucket=mybucket&prefix=myprefix&older-than=rel_time
+// - bucket is a mandatory query parameter
+// - prefix and older-than are optional query parameters
+// HTTP header x-minio-operation: list
+// ---------
+// Lists locks held on a given bucket, prefix and relative time.
+func (adminAPI adminAPIHandlers) ListLocksHandler(w http.ResponseWriter, r *http.Request) {
+	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponse(w, r, adminAPIErr, r.URL.Path)
+		return
+	}
+
+	vars := r.URL.Query()
+	bucket, prefix, relTime, adminAPIErr := validateLockQueryParams(vars)
+	if adminAPIErr != ErrNone {
+		writeErrorResponse(w, r, adminAPIErr, r.URL.Path)
+		return
+	}
+
+	// Fetch lock information of locks matching bucket/prefix that
+	// are available since relTime.
+	volLocks, err := listPeerLocksInfo(globalAdminPeers, bucket, prefix, relTime)
+	if err != nil {
+		writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		errorIf(err, "Failed to fetch lock information from remote nodes.")
+		return
+	}
+
+	// Marshal list of locks as json.
+	jsonBytes, err := json.Marshal(volLocks)
+	if err != nil {
+		writeErrorResponseNoHeader(w, r, ErrInternalError, r.URL.Path)
+		errorIf(err, "Failed to marshal lock information into json.")
+		return
+	}
+
+	// Reply with list of locks held on bucket, matching prefix
+	// older than relTime supplied, as json.
+	writeSuccessResponse(w, jsonBytes)
+}
+
+// ClearLocksHandler - POST /?lock&bucket=mybucket&prefix=myprefix&older-than=relTime
+// - bucket is a mandatory query parameter
+// - prefix and older-than are optional query parameters
+// HTTP header x-minio-operation: clear
+// ---------
+// Clear locks held on a given bucket, prefix and relative time.
+func (adminAPI adminAPIHandlers) ClearLocksHandler(w http.ResponseWriter, r *http.Request) {
+	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponse(w, r, adminAPIErr, r.URL.Path)
+		return
+	}
+
+	vars := r.URL.Query()
+	bucket, prefix, relTime, adminAPIErr := validateLockQueryParams(vars)
+	if adminAPIErr != ErrNone {
+		writeErrorResponse(w, r, adminAPIErr, r.URL.Path)
+		return
+	}
+
+	// Fetch lock information of locks matching bucket/prefix that
+	// are available since relTime.
+	volLocks, err := listPeerLocksInfo(globalAdminPeers, bucket, prefix, relTime)
+	if err != nil {
+		writeErrorResponseNoHeader(w, r, ErrInternalError, r.URL.Path)
+		errorIf(err, "Failed to fetch lock information from remote nodes.")
+		return
+	}
+
+	// Marshal list of locks as json.
+	jsonBytes, err := json.Marshal(volLocks)
+	if err != nil {
+		writeErrorResponseNoHeader(w, r, ErrInternalError, r.URL.Path)
+		errorIf(err, "Failed to marshal lock information into json.")
+		return
+	}
+	// Remove lock matching bucket/prefix older than relTime.
+	for _, volLock := range volLocks {
+		globalNSMutex.ForceUnlock(volLock.Bucket, volLock.Object)
+	}
+	// Reply with list of locks cleared, as json.
+	writeSuccessResponse(w, jsonBytes)
+}

cmd/admin-handlers_test.go

@@ -18,13 +18,17 @@ package cmd
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"testing"
 
 	router "github.com/gorilla/mux"
 )
 
+// cmdType - Represents different service subcomands like status, stop
+// and restart.
 type cmdType int
 
 const (
@@ -33,6 +37,7 @@ const (
 	restartCmd
 )
 
+// String - String representation for cmdType
 func (c cmdType) String() string {
 	switch c {
 	case statusCmd:
@@ -45,6 +50,8 @@ func (c cmdType) String() string {
 	return ""
 }
 
+// apiMethod - Returns the HTTP method corresponding to the admin REST
+// API for a given cmdType value.
 func (c cmdType) apiMethod() string {
 	switch c {
 	case statusCmd:
@@ -57,6 +64,8 @@ func (c cmdType) apiMethod() string {
 	return "GET"
 }
 
+// toServiceSignal - Helper function that translates a given cmdType
+// value to its corresponding serviceSignal value.
 func (c cmdType) toServiceSignal() serviceSignal {
 	switch c {
 	case statusCmd:
@@ -69,6 +78,8 @@ func (c cmdType) toServiceSignal() serviceSignal {
 	return serviceStatus
 }
 
+// testServiceSignalReceiver - Helper function that simulates a
+// go-routine waiting on service signal.
 func testServiceSignalReceiver(cmd cmdType, t *testing.T) {
 	expectedCmd := cmd.toServiceSignal()
 	serviceCmd := <-globalServiceSignalCh
@@ -77,12 +88,19 @@ func testServiceSignalReceiver(cmd cmdType, t *testing.T) {
 	}
 }
 
-func getAdminCmdRequest(cmd cmdType, cred credential) (*http.Request, error) {
+// getServiceCmdRequest - Constructs a management REST API request for service
+// subcommands for a given cmdType value.
+func getServiceCmdRequest(cmd cmdType, cred credential) (*http.Request, error) {
 	req, err := newTestRequest(cmd.apiMethod(), "/?service", 0, nil)
 	if err != nil {
 		return nil, err
 	}
+
+	// minioAdminOpHeader is to identify the request as a
+	// management REST API request.
 	req.Header.Set(minioAdminOpHeader, cmd.String())
+
+	// management REST API uses signature V4 for authentication.
 	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
 	if err != nil {
 		return nil, err
@@ -90,18 +108,26 @@ func getAdminCmdRequest(cmd cmdType, cred credential) (*http.Request, error) {
 	return req, nil
 }
 
+// testServicesCmdHandler - parametrizes service subcommand tests on
+// cmdType value.
 func testServicesCmdHandler(cmd cmdType, t *testing.T) {
+	// Initialize configuration for access/secret credentials.
 	rootPath, err := newTestConfig("us-east-1")
 	if err != nil {
 		t.Fatalf("Unable to initialize server config. %s", err)
 	}
 	defer removeAll(rootPath)
 
-	// Initialize admin peers to make admin RPC calls.
+	// Initialize admin peers to make admin RPC calls. Note: In a
+	// single node setup, this degenerates to a simple function
+	// call under the hood.
 	eps, err := parseStorageEndpoints([]string{"http://localhost"})
 	if err != nil {
 		t.Fatalf("Failed to parse storage end point - %v", err)
 	}
+
+	// Set globalMinioAddr to be able to distinguish local endpoints from remote.
+	globalMinioAddr = eps[0].Host
 	initGlobalAdminPeers(eps)
 
 	if cmd == statusCmd {
@@ -128,7 +154,7 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	registerAdminRouter(adminRouter)
 
 	rec := httptest.NewRecorder()
-	req, err := getAdminCmdRequest(cmd, credentials)
+	req, err := getServiceCmdRequest(cmd, credentials)
 	if err != nil {
 		t.Fatalf("Failed to build service status request %v", err)
 	}
@@ -151,14 +177,223 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	}
 }
 
+// Test for service status management REST API.
 func TestServiceStatusHandler(t *testing.T) {
 	testServicesCmdHandler(statusCmd, t)
 }
 
+// Test for service stop management REST API.
 func TestServiceStopHandler(t *testing.T) {
 	testServicesCmdHandler(stopCmd, t)
 }
 
+// Test for service restart management REST API.
 func TestServiceRestartHandler(t *testing.T) {
 	testServicesCmdHandler(restartCmd, t)
 }
+
+// Test for locks list management REST API.
+func TestListLocksHandler(t *testing.T) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Unable to initialize server config. %s", err)
+	}
+	defer removeAll(rootPath)
+
+	// Initialize admin peers to make admin RPC calls.
+	eps, err := parseStorageEndpoints([]string{"http://localhost"})
+	if err != nil {
+		t.Fatalf("Failed to parse storage end point - %v", err)
+	}
+
+	// Set globalMinioAddr to be able to distinguish local endpoints from remote.
+	globalMinioAddr = eps[0].Host
+	initGlobalAdminPeers(eps)
+
+	testCases := []struct {
+		bucket         string
+		prefix         string
+		relTime        string
+		expectedStatus int
+	}{
+		// Test 1 - valid testcase
+		{
+			bucket:         "mybucket",
+			prefix:         "myobject",
+			relTime:        "1s",
+			expectedStatus: 200,
+		},
+		// Test 2 - invalid duration
+		{
+			bucket:         "mybucket",
+			prefix:         "myprefix",
+			relTime:        "invalidDuration",
+			expectedStatus: 400,
+		},
+		// Test 3 - invalid bucket name
+		{
+			bucket:         `invalid\\Bucket`,
+			prefix:         "myprefix",
+			relTime:        "1h",
+			expectedStatus: 400,
+		},
+		// Test 4 - invalid prefix
+		{
+			bucket:         "mybucket",
+			prefix:         `invalid\\Prefix`,
+			relTime:        "1h",
+			expectedStatus: 400,
+		},
+	}
+
+	adminRouter := router.NewRouter()
+	registerAdminRouter(adminRouter)
+
+	for i, test := range testCases {
+		queryStr := fmt.Sprintf("&bucket=%s&prefix=%s&older-than=%s", test.bucket, test.prefix, test.relTime)
+		req, err := newTestRequest("GET", "/?lock"+queryStr, 0, nil)
+		if err != nil {
+			t.Fatalf("Test %d - Failed to construct list locks request - %v", i+1, err)
+		}
+		req.Header.Set(minioAdminOpHeader, "list")
+
+		cred := serverConfig.GetCredential()
+		err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
+		if err != nil {
+			t.Fatalf("Test %d - Failed to sign list locks request - %v", i+1, err)
+		}
+		rec := httptest.NewRecorder()
+		adminRouter.ServeHTTP(rec, req)
+		if test.expectedStatus != rec.Code {
+			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.expectedStatus, rec.Code)
+		}
+	}
+}
+
+// Test for locks clear management REST API.
+func TestClearLocksHandler(t *testing.T) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Unable to initialize server config. %s", err)
+	}
+	defer removeAll(rootPath)
+
+	// Initialize admin peers to make admin RPC calls.
+	eps, err := parseStorageEndpoints([]string{"http://localhost"})
+	if err != nil {
+		t.Fatalf("Failed to parse storage end point - %v", err)
+	}
+	initGlobalAdminPeers(eps)
+
+	testCases := []struct {
+		bucket         string
+		prefix         string
+		relTime        string
+		expectedStatus int
+	}{
+		// Test 1 - valid testcase
+		{
+			bucket:         "mybucket",
+			prefix:         "myobject",
+			relTime:        "1s",
+			expectedStatus: 200,
+		},
+		// Test 2 - invalid duration
+		{
+			bucket:         "mybucket",
+			prefix:         "myprefix",
+			relTime:        "invalidDuration",
+			expectedStatus: 400,
+		},
+		// Test 3 - invalid bucket name
+		{
+			bucket:         `invalid\\Bucket`,
+			prefix:         "myprefix",
+			relTime:        "1h",
+			expectedStatus: 400,
+		},
+		// Test 4 - invalid prefix
+		{
+			bucket:         "mybucket",
+			prefix:         `invalid\\Prefix`,
+			relTime:        "1h",
+			expectedStatus: 400,
+		},
+	}
+
+	adminRouter := router.NewRouter()
+	registerAdminRouter(adminRouter)
+
+	for i, test := range testCases {
+		queryStr := fmt.Sprintf("&bucket=%s&prefix=%s&older-than=%s", test.bucket, test.prefix, test.relTime)
+		req, err := newTestRequest("POST", "/?lock"+queryStr, 0, nil)
+		if err != nil {
+			t.Fatalf("Test %d - Failed to construct clear locks request - %v", i+1, err)
+		}
+		req.Header.Set(minioAdminOpHeader, "clear")
+
+		cred := serverConfig.GetCredential()
+		err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
+		if err != nil {
+			t.Fatalf("Test %d - Failed to sign clear locks request - %v", i+1, err)
+		}
+		rec := httptest.NewRecorder()
+		adminRouter.ServeHTTP(rec, req)
+		if test.expectedStatus != rec.Code {
+			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.expectedStatus, rec.Code)
+		}
+	}
+}
+
+// Test for lock query param validation helper function.
+func TestValidateLockQueryParams(t *testing.T) {
+	// Sample query values for test cases.
+	allValidVal := url.Values{}
+	allValidVal.Set(string(lockBucket), "bucket")
+	allValidVal.Set(string(lockPrefix), "prefix")
+	allValidVal.Set(string(lockOlderThan), "1s")
+
+	invalidBucketVal := url.Values{}
+	invalidBucketVal.Set(string(lockBucket), `invalid\\Bucket`)
+	invalidBucketVal.Set(string(lockPrefix), "prefix")
+	invalidBucketVal.Set(string(lockOlderThan), "invalidDuration")
+
+	invalidPrefixVal := url.Values{}
+	invalidPrefixVal.Set(string(lockBucket), "bucket")
+	invalidPrefixVal.Set(string(lockPrefix), `invalid\\PRefix`)
+	invalidPrefixVal.Set(string(lockOlderThan), "invalidDuration")
+
+	invalidOlderThanVal := url.Values{}
+	invalidOlderThanVal.Set(string(lockBucket), "bucket")
+	invalidOlderThanVal.Set(string(lockPrefix), "prefix")
+	invalidOlderThanVal.Set(string(lockOlderThan), "invalidDuration")
+
+	testCases := []struct {
+		qVals  url.Values
+		apiErr APIErrorCode
+	}{
+		{
+			qVals:  invalidBucketVal,
+			apiErr: ErrInvalidBucketName,
+		},
+		{
+			qVals:  invalidPrefixVal,
+			apiErr: ErrInvalidObjectName,
+		},
+		{
+			qVals:  invalidOlderThanVal,
+			apiErr: ErrInvalidDuration,
+		},
+		{
+			qVals:  allValidVal,
+			apiErr: ErrNone,
+		},
+	}
+
+	for i, test := range testCases {
+		_, _, _, apiErr := validateLockQueryParams(test.qVals)
+		if apiErr != test.apiErr {
+			t.Errorf("Test %d - Expected error %v but received %v", i+1, test.apiErr, apiErr)
+		}
+	}
+}

cmd/admin-router.go

@@ -29,7 +29,7 @@ func registerAdminRouter(mux *router.Router) {
 	// Admin router
 	adminRouter := mux.NewRoute().PathPrefix("/").Subrouter()
 
-	/// Admin operations
+	/// Service operations
 
 	// Service status
 	adminRouter.Methods("GET").Queries("service", "").Headers(minioAdminOpHeader, "status").HandlerFunc(adminAPI.ServiceStatusHandler)
@@ -37,4 +37,12 @@ func registerAdminRouter(mux *router.Router) {
 	adminRouter.Methods("POST").Queries("service", "").Headers(minioAdminOpHeader, "stop").HandlerFunc(adminAPI.ServiceStopHandler)
 	// Service restart
 	adminRouter.Methods("POST").Queries("service", "").Headers(minioAdminOpHeader, "restart").HandlerFunc(adminAPI.ServiceRestartHandler)
+
+	/// Lock operations
+
+	// List Locks
+	adminRouter.Methods("GET").Queries("lock", "").Headers(minioAdminOpHeader, "list").HandlerFunc(adminAPI.ListLocksHandler)
+
+	// Clear locks
+	adminRouter.Methods("POST").Queries("lock", "").Headers(minioAdminOpHeader, "clear").HandlerFunc(adminAPI.ClearLocksHandler)
 }

cmd/admin-rpc-client.go

@@ -20,6 +20,7 @@ import (
 	"net/url"
 	"path"
 	"sync"
+	"time"
 )
 
 // localAdminClient - represents admin operation to be executed locally.
@@ -32,11 +33,12 @@ type remoteAdminClient struct {
 	*AuthRPCClient
 }
 
-// stopRestarter - abstracts stop and restart operations for both
-// local and remote execution.
-type stopRestarter interface {
+// adminCmdRunner - abstracts local and remote execution of admin
+// commands like service stop and service restart.
+type adminCmdRunner interface {
 	Stop() error
 	Restart() error
+	ListLocks(bucket, prefix string, relTime time.Duration) ([]VolumeLockInfo, error)
 }
 
 // Stop - Sends a message over channel to the go-routine responsible
@@ -53,24 +55,43 @@ func (lc localAdminClient) Restart() error {
 	return nil
 }
 
+// ListLocks - Fetches lock information from local lock instrumentation.
+func (lc localAdminClient) ListLocks(bucket, prefix string, relTime time.Duration) ([]VolumeLockInfo, error) {
+	return listLocksInfo(bucket, prefix, relTime), nil
+}
+
 // Stop - Sends stop command to remote server via RPC.
 func (rc remoteAdminClient) Stop() error {
 	args := AuthRPCArgs{}
 	reply := AuthRPCReply{}
-	return rc.Call("Service.Shutdown", &args, &reply)
+	return rc.Call("Admin.Shutdown", &args, &reply)
 }
 
 // Restart - Sends restart command to remote server via RPC.
 func (rc remoteAdminClient) Restart() error {
 	args := AuthRPCArgs{}
 	reply := AuthRPCReply{}
-	return rc.Call("Service.Restart", &args, &reply)
+	return rc.Call("Admin.Restart", &args, &reply)
+}
+
+// ListLocks - Sends list locks command to remote server via RPC.
+func (rc remoteAdminClient) ListLocks(bucket, prefix string, relTime time.Duration) ([]VolumeLockInfo, error) {
+	listArgs := ListLocksQuery{
+		bucket:  bucket,
+		prefix:  prefix,
+		relTime: relTime,
+	}
+	var reply ListLocksReply
+	if err := rc.Call("Admin.ListLocks", &listArgs, &reply); err != nil {
+		return nil, err
+	}
+	return reply.volLocks, nil
 }
 
 // adminPeer - represents an entity that implements Stop and Restart methods.
 type adminPeer struct {
-	addr    string
-	svcClnt stopRestarter
+	addr      string
+	cmdRunner adminCmdRunner
 }
 
 // type alias for a collection of adminPeer.
@@ -105,13 +126,13 @@ func makeAdminPeers(eps []*url.URL) adminPeers {
 				secretKey:       serverCred.SecretKey,
 				serverAddr:      ep.Host,
 				secureConn:      isSSL(),
-				serviceEndpoint: path.Join(reservedBucket, servicePath),
-				serviceName:     "Service",
+				serviceEndpoint: path.Join(reservedBucket, adminPath),
+				serviceName:     "Admin",
 			}
 
 			servicePeers = append(servicePeers, adminPeer{
-				addr:    ep.Host,
-				svcClnt: &remoteAdminClient{newAuthRPCClient(cfg)},
+				addr:      ep.Host,
+				cmdRunner: &remoteAdminClient{newAuthRPCClient(cfg)},
 			})
 			seenAddr[ep.Host] = true
 		}
@@ -129,9 +150,9 @@ func initGlobalAdminPeers(eps []*url.URL) {
 func invokeServiceCmd(cp adminPeer, cmd serviceSignal) (err error) {
 	switch cmd {
 	case serviceStop:
-		err = cp.svcClnt.Stop()
+		err = cp.cmdRunner.Stop()
 	case serviceRestart:
-		err = cp.svcClnt.Restart()
+		err = cp.cmdRunner.Restart()
 	}
 	return err
 }
@@ -147,9 +168,58 @@ func sendServiceCmd(cps adminPeers, cmd serviceSignal) {
 		wg.Add(1)
 		go func(idx int) {
 			defer wg.Done()
-			errs[idx] = invokeServiceCmd(remotePeers[idx], cmd)
+			// we use idx+1 because remotePeers slice is 1 position shifted w.r.t cps
+			errs[idx+1] = invokeServiceCmd(remotePeers[idx], cmd)
 		}(i)
 	}
 	wg.Wait()
 	errs[0] = invokeServiceCmd(cps[0], cmd)
 }
+
+func listPeerLocksInfo(peers adminPeers, bucket, prefix string, relTime time.Duration) ([]VolumeLockInfo, error) {
+	// Used to aggregate volume lock information from all nodes.
+	allLocks := make([][]VolumeLockInfo, len(peers))
+	errs := make([]error, len(peers))
+	var wg sync.WaitGroup
+	localPeer := peers[0]
+	remotePeers := peers[1:]
+	for i, remotePeer := range remotePeers {
+		wg.Add(1)
+		go func(idx int, remotePeer adminPeer) {
+			defer wg.Done()
+			// `remotePeers` is right-shifted by one position relative to `peers`
+			allLocks[idx], errs[idx] = remotePeer.cmdRunner.ListLocks(bucket, prefix, relTime)
+		}(i+1, remotePeer)
+	}
+	wg.Wait()
+	allLocks[0], errs[0] = localPeer.cmdRunner.ListLocks(bucket, prefix, relTime)
+
+	// Summarizing errors received for ListLocks RPC across all
+	// nodes.  N B the possible unavailability of quorum in errors
+	// applies only to distributed setup.
+	errCount, err := reduceErrs(errs, []error{})
+	if err != nil {
+		if errCount >= (len(peers)/2 + 1) {
+			return nil, err
+		}
+		return nil, InsufficientReadQuorum{}
+	}
+
+	// Group lock information across nodes by (bucket, object)
+	// pair. For readability only.
+	paramLockMap := make(map[nsParam][]VolumeLockInfo)
+	for _, nodeLocks := range allLocks {
+		for _, lockInfo := range nodeLocks {
+			param := nsParam{
+				volume: lockInfo.Bucket,
+				path:   lockInfo.Object,
+			}
+			paramLockMap[param] = append(paramLockMap[param], lockInfo)
+		}
+	}
+	groupedLockInfos := []VolumeLockInfo{}
+	for _, volLocks := range paramLockMap {
+		groupedLockInfos = append(groupedLockInfos, volLocks...)
+	}
+	return groupedLockInfos, nil
+}

cmd/admin-rpc-server.go

@@ -18,20 +18,35 @@ package cmd
 
 import (
 	"net/rpc"
+	"time"
 
 	router "github.com/gorilla/mux"
 )
 
-const servicePath = "/admin/service"
+const adminPath = "/admin"
 
-// serviceCmd - exports RPC methods for service status, stop and
+// adminCmd - exports RPC methods for service status, stop and
 // restart commands.
-type serviceCmd struct {
+type adminCmd struct {
 	AuthRPCServer
 }
 
+// ListLocksQuery - wraps ListLocks API's query values to send over RPC.
+type ListLocksQuery struct {
+	AuthRPCArgs
+	bucket  string
+	prefix  string
+	relTime time.Duration
+}
+
+// ListLocksReply - wraps ListLocks response over RPC.
+type ListLocksReply struct {
+	AuthRPCReply
+	volLocks []VolumeLockInfo
+}
+
 // Shutdown - Shutdown this instance of minio server.
-func (s *serviceCmd) Shutdown(args *AuthRPCArgs, reply *AuthRPCReply) error {
+func (s *adminCmd) Shutdown(args *AuthRPCArgs, reply *AuthRPCReply) error {
 	if err := args.IsAuthenticated(); err != nil {
 		return err
 	}
@@ -41,7 +56,7 @@ func (s *serviceCmd) Shutdown(args *AuthRPCArgs, reply *AuthRPCReply) error {
 }
 
 // Restart - Restart this instance of minio server.
-func (s *serviceCmd) Restart(args *AuthRPCArgs, reply *AuthRPCReply) error {
+func (s *adminCmd) Restart(args *AuthRPCArgs, reply *AuthRPCReply) error {
 	if err := args.IsAuthenticated(); err != nil {
 		return err
 	}
@@ -50,16 +65,23 @@ func (s *serviceCmd) Restart(args *AuthRPCArgs, reply *AuthRPCReply) error {
 	return nil
 }
 
+// ListLocks - lists locks held by requests handled by this server instance.
+func (s *adminCmd) ListLocks(query *ListLocksQuery, reply *ListLocksReply) error {
+	volLocks := listLocksInfo(query.bucket, query.prefix, query.relTime)
+	*reply = ListLocksReply{volLocks: volLocks}
+	return nil
+}
+
 // registerAdminRPCRouter - registers RPC methods for service status,
 // stop and restart commands.
 func registerAdminRPCRouter(mux *router.Router) error {
-	adminRPCHandler := &serviceCmd{}
+	adminRPCHandler := &adminCmd{}
 	adminRPCServer := rpc.NewServer()
-	err := adminRPCServer.RegisterName("Service", adminRPCHandler)
+	err := adminRPCServer.RegisterName("Admin", adminRPCHandler)
 	if err != nil {
 		return traceError(err)
 	}
 	adminRouter := mux.NewRoute().PathPrefix(reservedBucket).Subrouter()
-	adminRouter.Path(servicePath).Handler(adminRPCServer)
+	adminRouter.Path(adminPath).Handler(adminRPCServer)
 	return nil
 }

cmd/admin-rpc-server_test.go

@@ -28,7 +28,7 @@ func testAdminCmd(cmd cmdType, t *testing.T) {
 	}
 	defer removeAll(rootPath)
 
-	adminServer := serviceCmd{}
+	adminServer := adminCmd{}
 	creds := serverConfig.GetCredential()
 	args := LoginRPCArgs{
 		Username:    creds.AccessKey,

cmd/api-errors.go

@@ -110,6 +110,7 @@ const (
 	ErrInvalidQuerySignatureAlgo
 	ErrInvalidQueryParams
 	ErrBucketAlreadyOwnedByYou
+	ErrInvalidDuration
 	// Add new error codes here.
 
 	// Bucket notification related errors.
@@ -477,6 +478,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Your previous request to create the named bucket succeeded and you already own it.",
 		HTTPStatusCode: http.StatusConflict,
 	},
+	ErrInvalidDuration: {
+		Code:           "InvalidDuration",
+		Description:    "Relative duration provided in the request is invalid.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 
 	/// Bucket notification related errors.
 	ErrEventNotification: {

cmd/lockinfo-handlers.go

@@ -16,7 +16,10 @@
 
 package cmd
 
-import "time"
+import (
+	"strings"
+	"time"
+)
 
 // SystemLockState - Structure to fill the lock state of entire object storage.
 // That is the total locks held, total calls blocked on locks and state of all the locks for the entire system.
@@ -26,7 +29,7 @@ type SystemLockState struct {
 	// be released.
 	TotalBlockedLocks int64 `json:"totalBlockedLocks"`
 	// Count of operations which has successfully acquired the lock but
-	// hasn't unlocked yet( operation in progress).
+	// hasn't unlocked yet (operation in progress).
 	TotalAcquiredLocks int64            `json:"totalAcquiredLocks"`
 	LocksInfoPerObject []VolumeLockInfo `json:"locksInfoPerObject"`
 }
@@ -64,11 +67,13 @@ func getSystemLockState() (SystemLockState, error) {
 	globalNSMutex.lockMapMutex.Lock()
 	defer globalNSMutex.lockMapMutex.Unlock()
 
-	lockState := SystemLockState{}
-
-	lockState.TotalBlockedLocks = globalNSMutex.counters.blocked
-	lockState.TotalLocks = globalNSMutex.counters.total
-	lockState.TotalAcquiredLocks = globalNSMutex.counters.granted
+	// Fetch current time once instead of fetching system time for every lock.
+	timeNow := time.Now().UTC()
+	lockState := SystemLockState{
+		TotalAcquiredLocks: globalNSMutex.counters.granted,
+		TotalLocks:         globalNSMutex.counters.total,
+		TotalBlockedLocks:  globalNSMutex.counters.blocked,
+	}
 
 	for param, debugLock := range globalNSMutex.debugLockMap {
 		volLockInfo := VolumeLockInfo{}
@@ -84,10 +89,57 @@ func getSystemLockState() (SystemLockState, error) {
 				LockType:    lockInfo.lType,
 				Status:      lockInfo.status,
 				Since:       lockInfo.since,
-				Duration:    time.Now().UTC().Sub(lockInfo.since),
+				Duration:    timeNow.Sub(lockInfo.since),
 			})
 		}
 		lockState.LocksInfoPerObject = append(lockState.LocksInfoPerObject, volLockInfo)
 	}
 	return lockState, nil
 }
+
+// listLocksInfo - Fetches locks held on bucket, matching prefix older than relTime.
+func listLocksInfo(bucket, prefix string, relTime time.Duration) []VolumeLockInfo {
+	globalNSMutex.lockMapMutex.Lock()
+	defer globalNSMutex.lockMapMutex.Unlock()
+
+	// Fetch current time once instead of fetching system time for every lock.
+	timeNow := time.Now().UTC()
+	volumeLocks := []VolumeLockInfo{}
+
+	for param, debugLock := range globalNSMutex.debugLockMap {
+		if param.volume != bucket {
+			continue
+		}
+		// N B empty prefix matches all param.path.
+		if !strings.HasPrefix(param.path, prefix) {
+			continue
+		}
+
+		volLockInfo := VolumeLockInfo{
+			Bucket:                param.volume,
+			Object:                param.path,
+			LocksOnObject:         debugLock.counters.total,
+			TotalBlockedLocks:     debugLock.counters.blocked,
+			LocksAcquiredOnObject: debugLock.counters.granted,
+		}
+		// Filter locks that are held on bucket, prefix.
+		for opsID, lockInfo := range debugLock.lockInfo {
+			elapsed := timeNow.Sub(lockInfo.since)
+			if elapsed < relTime {
+				continue
+			}
+			// Add locks that are older than relTime.
+			volLockInfo.LockDetailsOnObject = append(volLockInfo.LockDetailsOnObject,
+				OpsLockState{
+					OperationID: opsID,
+					LockSource:  lockInfo.lockSource,
+					LockType:    lockInfo.lType,
+					Status:      lockInfo.status,
+					Since:       lockInfo.since,
+					Duration:    elapsed,
+				})
+			volumeLocks = append(volumeLocks, volLockInfo)
+		}
+	}
+	return volumeLocks
+}

cmd/lockinfo-handlers_test.go

@@ -0,0 +1,81 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"testing"
+	"time"
+)
+
+// TestListLocksInfo - Test for listLocksInfo.
+func TestListLocksInfo(t *testing.T) {
+	// Initialize globalNSMutex to validate listing of lock
+	// instrumentation information.
+	isDistXL := false
+	initNSLock(isDistXL)
+
+	// Acquire a few locks to populate lock instrumentation.
+	// Take 10 read locks on bucket1/prefix1/obj1
+	for i := 0; i < 10; i++ {
+		readLk := globalNSMutex.NewNSLock("bucket1", "prefix1/obj1")
+		readLk.RLock()
+	}
+
+	// Take write locks on bucket1/prefix/obj{11..19}
+	for i := 0; i < 10; i++ {
+		wrLk := globalNSMutex.NewNSLock("bucket1", fmt.Sprintf("prefix1/obj%d", 10+i))
+		wrLk.Lock()
+	}
+
+	testCases := []struct {
+		bucket   string
+		prefix   string
+		relTime  time.Duration
+		numLocks int
+	}{
+		// Test 1 - Matches all the locks acquired above.
+		{
+			bucket:   "bucket1",
+			prefix:   "prefix1",
+			relTime:  time.Duration(0 * time.Second),
+			numLocks: 20,
+		},
+		// Test 2 - Bucket doesn't match.
+		{
+			bucket:   "bucket",
+			prefix:   "prefix1",
+			relTime:  time.Duration(0 * time.Second),
+			numLocks: 0,
+		},
+		// Test 3 - Prefix doesn't match.
+		{
+			bucket:   "bucket1",
+			prefix:   "prefix11",
+			relTime:  time.Duration(0 * time.Second),
+			numLocks: 0,
+		},
+	}
+
+	for i, test := range testCases {
+		actual := listLocksInfo(test.bucket, test.prefix, test.relTime)
+		if len(actual) != test.numLocks {
+			t.Errorf("Test %d - Expected %d locks but observed %d locks",
+				i+1, test.numLocks, len(actual))
+		}
+	}
+}

cmd/object-api-utils.go

@@ -49,7 +49,7 @@ var isIPAddress = regexp.MustCompile(`^(\d+\.){3}\d+$`)
 // See: http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
 func IsValidBucketName(bucket string) bool {
 	// Special case when bucket is equal to 'metaBucket'.
-	if bucket == minioMetaBucket {
+	if bucket == minioMetaBucket || bucket == minioMetaMultipartBucket {
 		return true
 	}
 	if len(bucket) < 3 || len(bucket) > 63 {