+ * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.lock; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +/** + * Lock implementation which also maintains counter. + */ +public final class ActiveLock { + + private Lock lock; + private AtomicInteger count; + + /** + * Use ActiveLock#newInstance to create instance. + */ + private ActiveLock() { + this.lock = new ReentrantLock(); + this.count = new AtomicInteger(0); + } + + /** + * Creates a new instance of ActiveLock. + * + * @return new ActiveLock + */ + public static ActiveLock newInstance() { + return new ActiveLock(); + } + + /** + * Acquires the lock. + * + *
If the lock is not available then the current thread becomes + * disabled for thread scheduling purposes and lies dormant until the + * lock has been acquired. + */ + public void lock() { + lock.lock(); + } + + /** + * Releases the lock. + */ + public void unlock() { + lock.unlock(); + } + + /** + * Increment the active count of the lock. + */ + void incrementActiveCount() { + count.incrementAndGet(); + } + + /** + * Decrement the active count of the lock. + */ + void decrementActiveCount() { + count.decrementAndGet(); + } + + /** + * Returns the active count on the lock. + * + * @return Number of active leases on the lock. + */ + int getActiveLockCount() { + return count.get(); + } + + /** + * Resets the active count on the lock. + */ + void resetCounter() { + count.set(0); + } + + @Override + public String toString() { + return lock.toString(); + } +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/LockManager.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/LockManager.java new file mode 100644 index 00000000000..49cf544626b --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/LockManager.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *
+ * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.lock;
+
+import org.apache.commons.pool2.impl.GenericObjectPool;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Manages the locks on a given resource. A new lock is created for each
+ * and every unique resource. Uniqueness of resource depends on the
+ * {@code equals} implementation of it.
+ */
+public class LockManager If the lock is not available then the current thread becomes
+ * disabled for thread scheduling purposes and lies dormant until the
+ * lock has been acquired.
+ */
+ public void lock(T resource) {
+ activeLocks.compute(resource, (k, v) -> {
+ ActiveLock lock;
+ try {
+ if (v == null) {
+ lock = lockPool.borrowObject();
+ } else {
+ lock = v;
+ }
+ lock.incrementActiveCount();
+ } catch (Exception ex) {
+ LOG.error("Unable to obtain lock.", ex);
+ throw new RuntimeException(ex);
+ }
+ return lock;
+ }).lock();
+ }
+
+ /**
+ * Releases the lock on given resource.
+ */
+ public void unlock(T resource) {
+ ActiveLock lock = activeLocks.get(resource);
+ if (lock == null) {
+ // Someone is releasing a lock which was never acquired. Log and return.
+ LOG.warn("Trying to release the lock on {}, which was never acquired.",
+ resource);
+ return;
+ }
+ lock.unlock();
+ activeLocks.computeIfPresent(resource, (k, v) -> {
+ v.decrementActiveCount();
+ if (v.getActiveLockCount() != 0) {
+ return v;
+ }
+ lockPool.returnObject(v);
+ return null;
+ });
+ }
+
+}
\ No newline at end of file
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/PooledLockFactory.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/PooledLockFactory.java
new file mode 100644
index 00000000000..4c24ef74b28
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/PooledLockFactory.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.lock;
+
+import org.apache.commons.pool2.BasePooledObjectFactory;
+import org.apache.commons.pool2.PooledObject;
+import org.apache.commons.pool2.impl.DefaultPooledObject;
+
+/**
+ * Pool factory to create {@code ActiveLock} instances.
+ */
+public class PooledLockFactory extends BasePooledObjectFactory
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.lock;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Test-cases to test LockManager.
+ */
+public class TestLockManager {
+
+ @Test(timeout = 1000)
+ public void testWithDifferentResource() {
+ LockManager
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ozone.lock.LockManager;
+
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_USER_PREFIX;
+
+/**
+ * Provides different locks to handle concurrency in OzoneMaster.
+ * We also maintain lock hierarchy, based on the weight.
+ *
+ *
+ * For example:
+ * If the lock is not available then the current thread becomes
+ * disabled for thread scheduling purposes and lies dormant until the
+ * lock has been acquired.
+ *
+ * @param user User on which the lock has to be acquired
+ */
+ public void acquireUserLock(String user) {
+ // Calling thread should not hold any volume or bucket lock.
+ if (hasAnyVolumeLock() || hasAnyBucketLock()) {
+ throw new RuntimeException(
+ "Thread '" + Thread.currentThread().getName() +
+ "' cannot acquire user lock" +
+ " while holding volume/bucket lock(s).");
+ }
+ manager.lock(OM_USER_PREFIX + user);
+ }
+
+ /**
+ * Releases the user lock on given resource.
+ */
+ public void releaseUserLock(String user) {
+ manager.unlock(OM_USER_PREFIX + user);
+ }
+
+ /**
+ * Acquires volume lock on the given resource.
+ *
+ * If the lock is not available then the current thread becomes
+ * disabled for thread scheduling purposes and lies dormant until the
+ * lock has been acquired.
+ *
+ * @param volume Volume on which the lock has to be acquired
+ */
+ public void acquireVolumeLock(String volume) {
+ // Calling thread should not hold any bucket lock.
+ if (hasAnyBucketLock()) {
+ throw new RuntimeException(
+ "Thread '" + Thread.currentThread().getName() +
+ "' cannot acquire volume lock while holding bucket lock(s).");
+ }
+ manager.lock(OM_KEY_PREFIX + volume);
+ myLocks.get().get(VOLUME_LOCK).incrementAndGet();
+ }
+
+ /**
+ * Releases the volume lock on given resource.
+ */
+ public void releaseVolumeLock(String volume) {
+ manager.unlock(OM_KEY_PREFIX + volume);
+ myLocks.get().get(VOLUME_LOCK).decrementAndGet();
+ }
+
+ /**
+ * Acquires bucket lock on the given resource.
+ *
+ * If the lock is not available then the current thread becomes
+ * disabled for thread scheduling purposes and lies dormant until the
+ * lock has been acquired.
+ *
+ * @param bucket Bucket on which the lock has to be acquired
+ */
+ public void acquireBucketLock(String volume, String bucket) {
+ manager.lock(OM_KEY_PREFIX + volume + OM_KEY_PREFIX + bucket);
+ myLocks.get().get(BUCKET_LOCK).incrementAndGet();
+ }
+
+
+ /**
+ * Releases the bucket lock on given resource.
+ */
+ public void releaseBucketLock(String volume, String bucket) {
+ manager.unlock(OM_KEY_PREFIX + volume + OM_KEY_PREFIX + bucket);
+ myLocks.get().get(BUCKET_LOCK).decrementAndGet();
+ }
+
+ /**
+ * Returns true if the current thread holds any volume lock.
+ * @return true if current thread holds volume lock, else false
+ */
+ private boolean hasAnyVolumeLock() {
+ return myLocks.get().get(VOLUME_LOCK).get() != 0;
+ }
+
+ /**
+ * Returns true if the current thread holds any bucket lock.
+ * @return true if current thread holds bucket lock, else false
+ */
+ private boolean hasAnyBucketLock() {
+ return myLocks.get().get(BUCKET_LOCK).get() != 0;
+ }
+
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeManagerImpl.java
index cf25215da78..c232bf1be75 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeManagerImpl.java
@@ -126,7 +126,8 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public void createVolume(OmVolumeArgs args) throws IOException {
Preconditions.checkNotNull(args);
- metadataManager.writeLock().lock();
+ metadataManager.getLock().acquireUserLock(args.getOwnerName());
+ metadataManager.getLock().acquireVolumeLock(args.getVolume());
try {
byte[] dbVolumeKey = metadataManager.getVolumeKey(args.getVolume());
byte[] volumeInfo = metadataManager.getVolumeTable().get(dbVolumeKey);
@@ -177,7 +178,8 @@ public class VolumeManagerImpl implements VolumeManager {
throw (IOException) ex;
}
} finally {
- metadataManager.writeLock().unlock();
+ metadataManager.getLock().releaseVolumeLock(args.getVolume());
+ metadataManager.getLock().releaseUserLock(args.getOwnerName());
}
}
@@ -192,7 +194,8 @@ public class VolumeManagerImpl implements VolumeManager {
public void setOwner(String volume, String owner) throws IOException {
Preconditions.checkNotNull(volume);
Preconditions.checkNotNull(owner);
- metadataManager.writeLock().lock();
+ metadataManager.getLock().acquireUserLock(owner);
+ metadataManager.getLock().acquireVolumeLock(volume);
try {
byte[] dbVolumeKey = metadataManager.getVolumeKey(volume);
byte[] volInfo = metadataManager.getVolumeTable().get(dbVolumeKey);
@@ -235,7 +238,8 @@ public class VolumeManagerImpl implements VolumeManager {
throw (IOException) ex;
}
} finally {
- metadataManager.writeLock().unlock();
+ metadataManager.getLock().releaseVolumeLock(volume);
+ metadataManager.getLock().releaseUserLock(owner);
}
}
@@ -248,7 +252,7 @@ public class VolumeManagerImpl implements VolumeManager {
*/
public void setQuota(String volume, long quota) throws IOException {
Preconditions.checkNotNull(volume);
- metadataManager.writeLock().lock();
+ metadataManager.getLock().acquireVolumeLock(volume);
try {
byte[] dbVolumeKey = metadataManager.getVolumeKey(volume);
byte[] volInfo = metadataManager.getVolumeTable().get(dbVolumeKey);
@@ -279,7 +283,7 @@ public class VolumeManagerImpl implements VolumeManager {
}
throw ex;
} finally {
- metadataManager.writeLock().unlock();
+ metadataManager.getLock().releaseVolumeLock(volume);
}
}
@@ -291,7 +295,7 @@ public class VolumeManagerImpl implements VolumeManager {
*/
public OmVolumeArgs getVolumeInfo(String volume) throws IOException {
Preconditions.checkNotNull(volume);
- metadataManager.readLock().lock();
+ metadataManager.getLock().acquireVolumeLock(volume);
try {
byte[] dbVolumeKey = metadataManager.getVolumeKey(volume);
byte[] volInfo = metadataManager.getVolumeTable().get(dbVolumeKey);
@@ -310,7 +314,7 @@ public class VolumeManagerImpl implements VolumeManager {
}
throw ex;
} finally {
- metadataManager.readLock().unlock();
+ metadataManager.getLock().releaseVolumeLock(volume);
}
}
@@ -323,7 +327,15 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public void deleteVolume(String volume) throws IOException {
Preconditions.checkNotNull(volume);
- metadataManager.writeLock().lock();
+ String owner;
+ metadataManager.getLock().acquireVolumeLock(volume);
+ try {
+ owner = getVolumeInfo(volume).getOwnerName();
+ } finally {
+ metadataManager.getLock().releaseVolumeLock(volume);
+ }
+ metadataManager.getLock().acquireUserLock(owner);
+ metadataManager.getLock().acquireVolumeLock(volume);
try {
byte[] dbVolumeKey = metadataManager.getVolumeKey(volume);
@@ -359,7 +371,8 @@ public class VolumeManagerImpl implements VolumeManager {
throw (IOException) ex;
}
} finally {
- metadataManager.writeLock().unlock();
+ metadataManager.getLock().releaseVolumeLock(volume);
+ metadataManager.getLock().releaseUserLock(owner);
}
}
@@ -375,7 +388,7 @@ public class VolumeManagerImpl implements VolumeManager {
throws IOException {
Preconditions.checkNotNull(volume);
Preconditions.checkNotNull(userAcl);
- metadataManager.readLock().lock();
+ metadataManager.getLock().acquireVolumeLock(volume);
try {
byte[] dbVolumeKey = metadataManager.getVolumeKey(volume);
byte[] volInfo = metadataManager.getVolumeTable().get(dbVolumeKey);
@@ -395,7 +408,7 @@ public class VolumeManagerImpl implements VolumeManager {
}
throw ex;
} finally {
- metadataManager.readLock().unlock();
+ metadataManager.getLock().releaseVolumeLock(volume);
}
}
@@ -405,12 +418,12 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public List
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Contains test-cases to verify OzoneManagerLock.
+ */
+public class TestOzoneManagerLock {
+
+ @Test(timeout = 1000)
+ public void testDifferentUserLock() {
+ OzoneManagerLock lock = new OzoneManagerLock(new OzoneConfiguration());
+ lock.acquireUserLock("userOne");
+ lock.acquireUserLock("userTwo");
+ lock.releaseUserLock("userOne");
+ lock.releaseUserLock("userTwo");
+ Assert.assertTrue(true);
+ }
+
+ @Test
+ public void testSameUserLock() throws Exception {
+ OzoneManagerLock lock = new OzoneManagerLock(new OzoneConfiguration());
+ lock.acquireUserLock("userOne");
+ AtomicBoolean gotLock = new AtomicBoolean(false);
+ new Thread(() -> {
+ lock.acquireUserLock("userOne");
+ gotLock.set(true);
+ lock.releaseUserLock("userOne");
+ }).start();
+ // Let's give some time for the new thread to run
+ Thread.sleep(100);
+ // Since the new thread is trying to get lock on same user, it will wait.
+ Assert.assertFalse(gotLock.get());
+ lock.releaseUserLock("userOne");
+ // Since we have released the lock, the new thread should have the lock
+ // now
+ // Let's give some time for the new thread to run
+ Thread.sleep(100);
+ Assert.assertTrue(gotLock.get());
+ }
+
+ @Test(timeout = 1000)
+ public void testDifferentVolumeLock() {
+ OzoneManagerLock lock = new OzoneManagerLock(new OzoneConfiguration());
+ lock.acquireVolumeLock("volOne");
+ lock.acquireVolumeLock("volTwo");
+ lock.releaseVolumeLock("volOne");
+ lock.releaseVolumeLock("volTwo");
+ Assert.assertTrue(true);
+ }
+
+ @Test
+ public void testSameVolumeLock() throws Exception {
+ OzoneManagerLock lock = new OzoneManagerLock(new OzoneConfiguration());
+ lock.acquireVolumeLock("volOne");
+ AtomicBoolean gotLock = new AtomicBoolean(false);
+ new Thread(() -> {
+ lock.acquireVolumeLock("volOne");
+ gotLock.set(true);
+ lock.releaseVolumeLock("volOne");
+ }).start();
+ // Let's give some time for the new thread to run
+ Thread.sleep(100);
+ // Since the new thread is trying to get lock on same user, it will wait.
+ Assert.assertFalse(gotLock.get());
+ lock.releaseVolumeLock("volOne");
+ // Since we have released the lock, the new thread should have the lock
+ // now
+ // Let's give some time for the new thread to run
+ Thread.sleep(100);
+ Assert.assertTrue(gotLock.get());
+ }
+
+ @Test(timeout = 1000)
+ public void testDifferentBucketLock() {
+ OzoneManagerLock lock = new OzoneManagerLock(new OzoneConfiguration());
+ lock.acquireBucketLock("volOne", "bucketOne");
+ lock.acquireBucketLock("volOne", "bucketTwo");
+ lock.releaseBucketLock("volOne", "bucketTwo");
+ lock.releaseBucketLock("volOne", "bucketOne");
+ Assert.assertTrue(true);
+ }
+
+ @Test
+ public void testSameBucketLock() throws Exception {
+ OzoneManagerLock lock = new OzoneManagerLock(new OzoneConfiguration());
+ lock.acquireBucketLock("volOne", "bucketOne");
+ AtomicBoolean gotLock = new AtomicBoolean(false);
+ new Thread(() -> {
+ lock.acquireBucketLock("volOne", "bucketOne");
+ gotLock.set(true);
+ lock.releaseBucketLock("volOne", "bucketOne");
+ }).start();
+ // Let's give some time for the new thread to run
+ Thread.sleep(100);
+ // Since the new thread is trying to get lock on same user, it will wait.
+ Assert.assertFalse(gotLock.get());
+ lock.releaseBucketLock("volOne", "bucketOne");
+ // Since we have released the lock, the new thread should have the lock
+ // now
+ // Let's give some time for the new thread to run
+ Thread.sleep(100);
+ Assert.assertTrue(gotLock.get());
+ }
+
+ @Test(timeout = 1000)
+ public void testVolumeLockAfterUserLock() {
+ OzoneManagerLock lock = new OzoneManagerLock(new OzoneConfiguration());
+ lock.acquireUserLock("userOne");
+ lock.acquireVolumeLock("volOne");
+ lock.releaseVolumeLock("volOne");
+ lock.releaseUserLock("userOne");
+ Assert.assertTrue(true);
+ }
+
+ @Test(timeout = 1000)
+ public void testBucketLockAfterVolumeLock() {
+ OzoneManagerLock lock = new OzoneManagerLock(new OzoneConfiguration());
+ lock.acquireVolumeLock("volOne");
+ lock.acquireBucketLock("volOne", "bucketOne");
+ lock.releaseBucketLock("volOne", "bucketOne");
+ lock.releaseVolumeLock("volOne");
+ Assert.assertTrue(true);
+ }
+
+ @Test(timeout = 1000)
+ public void testBucketLockAfterVolumeLockAfterUserLock() {
+ OzoneManagerLock lock = new OzoneManagerLock(new OzoneConfiguration());
+ lock.acquireUserLock("userOne");
+ lock.acquireVolumeLock("volOne");
+ lock.acquireBucketLock("volOne", "bucketOne");
+ lock.releaseBucketLock("volOne", "bucketOne");
+ lock.releaseVolumeLock("volOne");
+ lock.releaseUserLock("userOne");
+ Assert.assertTrue(true);
+ }
+
+ @Test
+ public void testUserLockAfterVolumeLock() {
+ OzoneManagerLock lock = new OzoneManagerLock(new OzoneConfiguration());
+ lock.acquireVolumeLock("volOne");
+ try {
+ lock.acquireUserLock("userOne");
+ Assert.fail();
+ } catch (RuntimeException ex) {
+ String msg =
+ "cannot acquire user lock while holding volume/bucket lock(s).";
+ Assert.assertTrue(ex.getMessage().contains(msg));
+ }
+ lock.releaseVolumeLock("volOne");
+ Assert.assertTrue(true);
+ }
+
+ @Test
+ public void testVolumeLockAfterBucketLock() {
+ OzoneManagerLock lock = new OzoneManagerLock(new OzoneConfiguration());
+ lock.acquireBucketLock("volOne", "bucketOne");
+ try {
+ lock.acquireVolumeLock("volOne");
+ Assert.fail();
+ } catch (RuntimeException ex) {
+ String msg =
+ "cannot acquire volume lock while holding bucket lock(s).";
+ Assert.assertTrue(ex.getMessage().contains(msg));
+ }
+ lock.releaseBucketLock("volOne", "bucketOne");
+ Assert.assertTrue(true);
+ }
+
+
+}
\ No newline at end of file
+ *
+ *
+ * One cannot obtain a lower weight lock while holding a lock with higher
+ * weight. The other way around is possible.
+ *
+ * WEIGHT LOCK
+ *
+ *
+ * 0 User Lock
+ *
+ *
+ * 1 Volume Lock
+ *
+ *
+ * 2 Bucket Lock
+ *
+ *
+ *
+ * -> acquireVolumeLock (will work)
+ * +-> acquireBucketLock (will work)
+ * +--> acquireUserLock (will throw Exception)
+ *
+ *
+ * To acquire a user lock you should not hold any Volume/Bucket lock. Similarly
+ * to acquire a Volume lock you should not hold any Bucket lock.
+ *
+ */
+public final class OzoneManagerLock {
+
+ private static final String VOLUME_LOCK = "volumeLock";
+ private static final String BUCKET_LOCK = "bucketLock";
+
+
+ private final LockManager