From b43cde2ce19255f293983f2e4c41c79bae6bd66c Mon Sep 17 00:00:00 2001 From: zhangduo Date: Sun, 29 Apr 2018 20:04:24 +0800 Subject: [PATCH] HBASE-20507 Do not need to call recoverLease on the broken file when we fail to create a wal writer --- .../FanOutOneBlockAsyncDFSOutputHelper.java | 3 - .../TestOverwriteFileUnderConstruction.java | 108 ++++++++++++++++++ 2 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestOverwriteFileUnderConstruction.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java index d7aa897d56b..34d9f29e3e7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java @@ -800,8 +800,6 @@ public final class FanOutOneBlockAsyncDFSOutputHelper { } else { throw e.unwrapRemoteException(); } - } catch (NameNodeException e) { - throw e; } catch (IOException e) { LOG.warn("create fan-out dfs output {} failed, retry = {}", src, retry, e); if (retry >= createMaxRetries) { @@ -830,7 +828,6 @@ public final class FanOutOneBlockAsyncDFSOutputHelper { } } endFileLease(client, stat.getFileId()); - fsUtils.recoverFileLease(dfs, new Path(src), conf, new CancelOnClose(client)); } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestOverwriteFileUnderConstruction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestOverwriteFileUnderConstruction.java new file mode 100644 index 00000000000..7f75fdb2680 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestOverwriteFileUnderConstruction.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.asyncfs; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +import java.io.IOException; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException; +import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException; +import org.apache.hadoop.ipc.RemoteException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + +/** + * Used to confirm that it is OK to overwrite a file which is being written currently. + */ +@Category({ MiscTests.class, MediumTests.class }) +public class TestOverwriteFileUnderConstruction { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestOverwriteFileUnderConstruction.class); + + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + private static FileSystem FS; + + @Rule + public final TestName name = new TestName(); + + @BeforeClass + public static void setUp() throws Exception { + UTIL.startMiniDFSCluster(3); + FS = UTIL.getDFSCluster().getFileSystem(); + } + + @AfterClass + public static void tearDown() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Test + public void testNotOverwrite() throws IOException { + Path file = new Path("/" + name.getMethodName()); + try (FSDataOutputStream out1 = FS.create(file)) { + try { + FS.create(file, false); + fail("Should fail as there is a file with the same name which is being written"); + } catch (RemoteException e) { + // expected + assertThat(e.unwrapRemoteException(), instanceOf(AlreadyBeingCreatedException.class)); + } + } + } + + @Test + public void testOverwrite() throws IOException { + Path file = new Path("/" + name.getMethodName()); + FSDataOutputStream out1 = FS.create(file); + FSDataOutputStream out2 = FS.create(file, true); + out1.write(2); + out2.write(1); + try { + out1.close(); + // a successful close is also OK for us so no assertion here, we just need to confirm that the + // data in the file are correct. + } catch (RemoteException e) { + // expected + assertThat(e.unwrapRemoteException(), instanceOf(LeaseExpiredException.class)); + } + out2.close(); + try (FSDataInputStream in = FS.open(file)) { + assertEquals(1, in.read()); + assertEquals(-1, in.read()); + } + } +}