HADOOP-18155. Refactor tests in TestFileUtil (#4053)

This commit is contained in:
Gautham B A 2022-03-10 22:02:38 +05:30 committed by GitHub
parent 383b73417d
commit d0fa9b5775
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 275 additions and 163 deletions

View File

@ -39,6 +39,7 @@ import java.nio.file.AccessDeniedException;
import java.nio.file.FileSystems; import java.nio.file.FileSystems;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.LinkOption; import java.nio.file.LinkOption;
import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Enumeration; import java.util.Enumeration;
import java.util.List; import java.util.List;
@ -992,6 +993,14 @@ public class FileUtil {
+ " would create entry outside of " + outputDir); + " would create entry outside of " + outputDir);
} }
if (entry.isSymbolicLink() || entry.isLink()) {
String canonicalTargetPath = getCanonicalPath(entry.getLinkName(), outputDir);
if (!canonicalTargetPath.startsWith(targetDirPath)) {
throw new IOException(
"expanding " + entry.getName() + " would create entry outside of " + outputDir);
}
}
if (entry.isDirectory()) { if (entry.isDirectory()) {
File subDir = new File(outputDir, entry.getName()); File subDir = new File(outputDir, entry.getName());
if (!subDir.mkdirs() && !subDir.isDirectory()) { if (!subDir.mkdirs() && !subDir.isDirectory()) {
@ -1007,10 +1016,12 @@ public class FileUtil {
} }
if (entry.isSymbolicLink()) { if (entry.isSymbolicLink()) {
// Create symbolic link relative to tar parent dir // Create symlink with canonical target path to ensure that we don't extract
Files.createSymbolicLink(FileSystems.getDefault() // outside targetDirPath
.getPath(outputDir.getPath(), entry.getName()), String canonicalTargetPath = getCanonicalPath(entry.getLinkName(), outputDir);
FileSystems.getDefault().getPath(entry.getLinkName())); Files.createSymbolicLink(
FileSystems.getDefault().getPath(outputDir.getPath(), entry.getName()),
FileSystems.getDefault().getPath(canonicalTargetPath));
return; return;
} }
@ -1022,7 +1033,8 @@ public class FileUtil {
} }
if (entry.isLink()) { if (entry.isLink()) {
File src = new File(outputDir, entry.getLinkName()); String canonicalTargetPath = getCanonicalPath(entry.getLinkName(), outputDir);
File src = new File(canonicalTargetPath);
HardLink.createHardLink(src, outputFile); HardLink.createHardLink(src, outputFile);
return; return;
} }
@ -1030,6 +1042,20 @@ public class FileUtil {
org.apache.commons.io.FileUtils.copyToFile(tis, outputFile); org.apache.commons.io.FileUtils.copyToFile(tis, outputFile);
} }
/**
* Gets the canonical path for the given path.
*
* @param path The path for which the canonical path needs to be computed.
* @param parentDir The parent directory to use if the path is a relative path.
* @return The canonical path of the given path.
*/
private static String getCanonicalPath(String path, File parentDir) throws IOException {
java.nio.file.Path targetPath = Paths.get(path);
return (targetPath.isAbsolute() ?
new File(path) :
new File(parentDir, path)).getCanonicalPath();
}
/** /**
* Class for creating hardlinks. * Class for creating hardlinks.
* Supports Unix, WindXP. * Supports Unix, WindXP.

View File

@ -42,13 +42,14 @@ import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.net.UnknownHostException; import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.FileSystems;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Objects;
import java.util.jar.Attributes; import java.util.jar.Attributes;
import java.util.jar.JarFile; import java.util.jar.JarFile;
import java.util.jar.Manifest; import java.util.jar.Manifest;
@ -60,9 +61,12 @@ import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.LambdaTestUtils;
import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.StringUtils;
import org.apache.tools.tar.TarEntry; import org.apache.tools.tar.TarEntry;
import org.apache.tools.tar.TarOutputStream; import org.apache.tools.tar.TarOutputStream;
import org.assertj.core.api.Assertions;
import org.junit.After; import org.junit.After;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
@ -158,13 +162,12 @@ public class TestFileUtil {
FileUtils.forceMkdir(dir1); FileUtils.forceMkdir(dir1);
FileUtils.forceMkdir(dir2); FileUtils.forceMkdir(dir2);
new File(del, FILE).createNewFile(); Verify.createNewFile(new File(del, FILE));
File tmpFile = new File(tmp, FILE); File tmpFile = Verify.createNewFile(new File(tmp, FILE));
tmpFile.createNewFile();
// create files // create files
new File(dir1, FILE).createNewFile(); Verify.createNewFile(new File(dir1, FILE));
new File(dir2, FILE).createNewFile(); Verify.createNewFile(new File(dir2, FILE));
// create a symlink to file // create a symlink to file
File link = new File(del, LINK); File link = new File(del, LINK);
@ -173,7 +176,7 @@ public class TestFileUtil {
// create a symlink to dir // create a symlink to dir
File linkDir = new File(del, "tmpDir"); File linkDir = new File(del, "tmpDir");
FileUtil.symLink(tmp.toString(), linkDir.toString()); FileUtil.symLink(tmp.toString(), linkDir.toString());
Assert.assertEquals(5, del.listFiles().length); Assert.assertEquals(5, Objects.requireNonNull(del.listFiles()).length);
// create files in partitioned directories // create files in partitioned directories
createFile(partitioned, "part-r-00000", "foo"); createFile(partitioned, "part-r-00000", "foo");
@ -200,13 +203,9 @@ public class TestFileUtil {
private File createFile(File directory, String name, String contents) private File createFile(File directory, String name, String contents)
throws IOException { throws IOException {
File newFile = new File(directory, name); File newFile = new File(directory, name);
PrintWriter pw = new PrintWriter(newFile); try (PrintWriter pw = new PrintWriter(newFile)) {
try {
pw.println(contents); pw.println(contents);
} }
finally {
pw.close();
}
return newFile; return newFile;
} }
@ -218,11 +217,11 @@ public class TestFileUtil {
//Test existing directory with no files case //Test existing directory with no files case
File newDir = new File(tmp.getPath(),"test"); File newDir = new File(tmp.getPath(),"test");
newDir.mkdir(); Verify.mkdir(newDir);
Assert.assertTrue("Failed to create test dir", newDir.exists()); Assert.assertTrue("Failed to create test dir", newDir.exists());
files = FileUtil.listFiles(newDir); files = FileUtil.listFiles(newDir);
Assert.assertEquals(0, files.length); Assert.assertEquals(0, files.length);
newDir.delete(); assertTrue(newDir.delete());
Assert.assertFalse("Failed to delete test dir", newDir.exists()); Assert.assertFalse("Failed to delete test dir", newDir.exists());
//Test non-existing directory case, this throws //Test non-existing directory case, this throws
@ -244,11 +243,11 @@ public class TestFileUtil {
//Test existing directory with no files case //Test existing directory with no files case
File newDir = new File(tmp.getPath(),"test"); File newDir = new File(tmp.getPath(),"test");
newDir.mkdir(); Verify.mkdir(newDir);
Assert.assertTrue("Failed to create test dir", newDir.exists()); Assert.assertTrue("Failed to create test dir", newDir.exists());
files = FileUtil.list(newDir); files = FileUtil.list(newDir);
Assert.assertEquals("New directory unexpectedly contains files", 0, files.length); Assert.assertEquals("New directory unexpectedly contains files", 0, files.length);
newDir.delete(); assertTrue(newDir.delete());
Assert.assertFalse("Failed to delete test dir", newDir.exists()); Assert.assertFalse("Failed to delete test dir", newDir.exists());
//Test non-existing directory case, this throws //Test non-existing directory case, this throws
@ -266,7 +265,7 @@ public class TestFileUtil {
public void testFullyDelete() throws IOException { public void testFullyDelete() throws IOException {
boolean ret = FileUtil.fullyDelete(del); boolean ret = FileUtil.fullyDelete(del);
Assert.assertTrue(ret); Assert.assertTrue(ret);
Assert.assertFalse(del.exists()); Verify.notExists(del);
validateTmpDir(); validateTmpDir();
} }
@ -279,13 +278,13 @@ public class TestFileUtil {
@Test (timeout = 30000) @Test (timeout = 30000)
public void testFullyDeleteSymlinks() throws IOException { public void testFullyDeleteSymlinks() throws IOException {
File link = new File(del, LINK); File link = new File(del, LINK);
Assert.assertEquals(5, del.list().length); assertDelListLength(5);
// Since tmpDir is symlink to tmp, fullyDelete(tmpDir) should not // Since tmpDir is symlink to tmp, fullyDelete(tmpDir) should not
// delete contents of tmp. See setupDirs for details. // delete contents of tmp. See setupDirs for details.
boolean ret = FileUtil.fullyDelete(link); boolean ret = FileUtil.fullyDelete(link);
Assert.assertTrue(ret); Assert.assertTrue(ret);
Assert.assertFalse(link.exists()); Verify.notExists(link);
Assert.assertEquals(4, del.list().length); assertDelListLength(4);
validateTmpDir(); validateTmpDir();
File linkDir = new File(del, "tmpDir"); File linkDir = new File(del, "tmpDir");
@ -293,8 +292,8 @@ public class TestFileUtil {
// delete contents of tmp. See setupDirs for details. // delete contents of tmp. See setupDirs for details.
ret = FileUtil.fullyDelete(linkDir); ret = FileUtil.fullyDelete(linkDir);
Assert.assertTrue(ret); Assert.assertTrue(ret);
Assert.assertFalse(linkDir.exists()); Verify.notExists(linkDir);
Assert.assertEquals(3, del.list().length); assertDelListLength(3);
validateTmpDir(); validateTmpDir();
} }
@ -310,16 +309,16 @@ public class TestFileUtil {
// to make y as a dangling link to file tmp/x // to make y as a dangling link to file tmp/x
boolean ret = FileUtil.fullyDelete(tmp); boolean ret = FileUtil.fullyDelete(tmp);
Assert.assertTrue(ret); Assert.assertTrue(ret);
Assert.assertFalse(tmp.exists()); Verify.notExists(tmp);
// dangling symlink to file // dangling symlink to file
File link = new File(del, LINK); File link = new File(del, LINK);
Assert.assertEquals(5, del.list().length); assertDelListLength(5);
// Even though 'y' is dangling symlink to file tmp/x, fullyDelete(y) // Even though 'y' is dangling symlink to file tmp/x, fullyDelete(y)
// should delete 'y' properly. // should delete 'y' properly.
ret = FileUtil.fullyDelete(link); ret = FileUtil.fullyDelete(link);
Assert.assertTrue(ret); Assert.assertTrue(ret);
Assert.assertEquals(4, del.list().length); assertDelListLength(4);
// dangling symlink to directory // dangling symlink to directory
File linkDir = new File(del, "tmpDir"); File linkDir = new File(del, "tmpDir");
@ -327,22 +326,22 @@ public class TestFileUtil {
// delete tmpDir properly. // delete tmpDir properly.
ret = FileUtil.fullyDelete(linkDir); ret = FileUtil.fullyDelete(linkDir);
Assert.assertTrue(ret); Assert.assertTrue(ret);
Assert.assertEquals(3, del.list().length); assertDelListLength(3);
} }
@Test (timeout = 30000) @Test (timeout = 30000)
public void testFullyDeleteContents() throws IOException { public void testFullyDeleteContents() throws IOException {
boolean ret = FileUtil.fullyDeleteContents(del); boolean ret = FileUtil.fullyDeleteContents(del);
Assert.assertTrue(ret); Assert.assertTrue(ret);
Assert.assertTrue(del.exists()); Verify.exists(del);
Assert.assertEquals(0, del.listFiles().length); Assert.assertEquals(0, Objects.requireNonNull(del.listFiles()).length);
validateTmpDir(); validateTmpDir();
} }
private void validateTmpDir() { private void validateTmpDir() {
Assert.assertTrue(tmp.exists()); Verify.exists(tmp);
Assert.assertEquals(1, tmp.listFiles().length); Assert.assertEquals(1, Objects.requireNonNull(tmp.listFiles()).length);
Assert.assertTrue(new File(tmp, FILE).exists()); Verify.exists(new File(tmp, FILE));
} }
/** /**
@ -366,15 +365,15 @@ public class TestFileUtil {
* @throws IOException * @throws IOException
*/ */
private void setupDirsAndNonWritablePermissions() throws IOException { private void setupDirsAndNonWritablePermissions() throws IOException {
new MyFile(del, FILE_1_NAME).createNewFile(); Verify.createNewFile(new MyFile(del, FILE_1_NAME));
// "file1" is non-deletable by default, see MyFile.delete(). // "file1" is non-deletable by default, see MyFile.delete().
xSubDir.mkdirs(); Verify.mkdirs(xSubDir);
file2.createNewFile(); Verify.createNewFile(file2);
xSubSubDir.mkdirs(); Verify.mkdirs(xSubSubDir);
file22.createNewFile(); Verify.createNewFile(file22);
revokePermissions(file22); revokePermissions(file22);
revokePermissions(xSubSubDir); revokePermissions(xSubSubDir);
@ -382,8 +381,8 @@ public class TestFileUtil {
revokePermissions(file2); revokePermissions(file2);
revokePermissions(xSubDir); revokePermissions(xSubDir);
ySubDir.mkdirs(); Verify.mkdirs(ySubDir);
file3.createNewFile(); Verify.createNewFile(file3);
File tmpFile = new File(tmp, FILE); File tmpFile = new File(tmp, FILE);
tmpFile.createNewFile(); tmpFile.createNewFile();
@ -462,8 +461,8 @@ public class TestFileUtil {
boolean ret = FileUtil.fullyDelete(linkDir); boolean ret = FileUtil.fullyDelete(linkDir);
// fail symlink deletion // fail symlink deletion
Assert.assertFalse(ret); Assert.assertFalse(ret);
Assert.assertTrue(linkDir.exists()); Verify.exists(linkDir);
Assert.assertEquals(5, del.list().length); assertDelListLength(5);
// tmp dir should exist // tmp dir should exist
validateTmpDir(); validateTmpDir();
// simulate disk recovers and turns good // simulate disk recovers and turns good
@ -471,12 +470,94 @@ public class TestFileUtil {
ret = FileUtil.fullyDelete(linkDir); ret = FileUtil.fullyDelete(linkDir);
// success symlink deletion // success symlink deletion
Assert.assertTrue(ret); Assert.assertTrue(ret);
Assert.assertFalse(linkDir.exists()); Verify.notExists(linkDir);
Assert.assertEquals(4, del.list().length); assertDelListLength(4);
// tmp dir should exist // tmp dir should exist
validateTmpDir(); validateTmpDir();
} }
/**
* Asserts if the {@link TestFileUtil#del} meets the given expected length.
*
* @param expectedLength The expected length of the {@link TestFileUtil#del}.
*/
private void assertDelListLength(int expectedLength) {
Assertions.assertThat(del.list()).describedAs("del list").isNotNull().hasSize(expectedLength);
}
/**
* Helper class to perform {@link File} operation and also verify them.
*/
public static class Verify {
/**
* Invokes {@link File#createNewFile()} on the given {@link File} instance.
*
* @param file The file to call {@link File#createNewFile()} on.
* @return The result of {@link File#createNewFile()}.
* @throws IOException As per {@link File#createNewFile()}.
*/
public static File createNewFile(File file) throws IOException {
assertTrue("Unable to create new file " + file, file.createNewFile());
return file;
}
/**
* Invokes {@link File#mkdir()} on the given {@link File} instance.
*
* @param file The file to call {@link File#mkdir()} on.
* @return The result of {@link File#mkdir()}.
*/
public static File mkdir(File file) {
assertTrue("Unable to mkdir for " + file, file.mkdir());
return file;
}
/**
* Invokes {@link File#mkdirs()} on the given {@link File} instance.
*
* @param file The file to call {@link File#mkdirs()} on.
* @return The result of {@link File#mkdirs()}.
*/
public static File mkdirs(File file) {
assertTrue("Unable to mkdirs for " + file, file.mkdirs());
return file;
}
/**
* Invokes {@link File#delete()} on the given {@link File} instance.
*
* @param file The file to call {@link File#delete()} on.
* @return The result of {@link File#delete()}.
*/
public static File delete(File file) {
assertTrue("Unable to delete " + file, file.delete());
return file;
}
/**
* Invokes {@link File#exists()} on the given {@link File} instance.
*
* @param file The file to call {@link File#exists()} on.
* @return The result of {@link File#exists()}.
*/
public static File exists(File file) {
assertTrue("Expected file " + file + " doesn't exist", file.exists());
return file;
}
/**
* Invokes {@link File#exists()} on the given {@link File} instance to check if the
* {@link File} doesn't exists.
*
* @param file The file to call {@link File#exists()} on.
* @return The negation of the result of {@link File#exists()}.
*/
public static File notExists(File file) {
assertFalse("Expected file " + file + " must not exist", file.exists());
return file;
}
}
/** /**
* Extend {@link File}. Same as {@link File} except for two things: (1) This * Extend {@link File}. Same as {@link File} except for two things: (1) This
* treats file1Name as a very special file which is not delete-able * treats file1Name as a very special file which is not delete-able
@ -609,14 +690,13 @@ public class TestFileUtil {
FileUtil.chmod(partitioned.getAbsolutePath(), "0777", true/*recursive*/); FileUtil.chmod(partitioned.getAbsolutePath(), "0777", true/*recursive*/);
} }
} }
@Test (timeout = 30000) @Test (timeout = 30000)
public void testUnTar() throws IOException { public void testUnTar() throws Exception {
// make a simple tar: // make a simple tar:
final File simpleTar = new File(del, FILE); final File simpleTar = new File(del, FILE);
OutputStream os = new FileOutputStream(simpleTar); OutputStream os = new FileOutputStream(simpleTar);
TarOutputStream tos = new TarOutputStream(os); try (TarOutputStream tos = new TarOutputStream(os)) {
try {
TarEntry te = new TarEntry("/bar/foo"); TarEntry te = new TarEntry("/bar/foo");
byte[] data = "some-content".getBytes("UTF-8"); byte[] data = "some-content".getBytes("UTF-8");
te.setSize(data.length); te.setSize(data.length);
@ -625,55 +705,42 @@ public class TestFileUtil {
tos.closeEntry(); tos.closeEntry();
tos.flush(); tos.flush();
tos.finish(); tos.finish();
} finally {
tos.close();
} }
// successfully untar it into an existing dir: // successfully untar it into an existing dir:
FileUtil.unTar(simpleTar, tmp); FileUtil.unTar(simpleTar, tmp);
// check result: // check result:
assertTrue(new File(tmp, "/bar/foo").exists()); Verify.exists(new File(tmp, "/bar/foo"));
assertEquals(12, new File(tmp, "/bar/foo").length()); assertEquals(12, new File(tmp, "/bar/foo").length());
final File regularFile = new File(tmp, "QuickBrownFoxJumpsOverTheLazyDog"); final File regularFile =
regularFile.createNewFile(); Verify.createNewFile(new File(tmp, "QuickBrownFoxJumpsOverTheLazyDog"));
assertTrue(regularFile.exists()); LambdaTestUtils.intercept(IOException.class, () -> FileUtil.unTar(simpleTar, regularFile));
try {
FileUtil.unTar(simpleTar, regularFile);
assertTrue("An IOException expected.", false);
} catch (IOException ioe) {
// okay
}
} }
@Test (timeout = 30000) @Test (timeout = 30000)
public void testReplaceFile() throws IOException { public void testReplaceFile() throws IOException {
final File srcFile = new File(tmp, "src");
// src exists, and target does not exist: // src exists, and target does not exist:
srcFile.createNewFile(); final File srcFile = Verify.createNewFile(new File(tmp, "src"));
assertTrue(srcFile.exists());
final File targetFile = new File(tmp, "target"); final File targetFile = new File(tmp, "target");
assertTrue(!targetFile.exists()); Verify.notExists(targetFile);
FileUtil.replaceFile(srcFile, targetFile); FileUtil.replaceFile(srcFile, targetFile);
assertTrue(!srcFile.exists()); Verify.notExists(srcFile);
assertTrue(targetFile.exists()); Verify.exists(targetFile);
// src exists and target is a regular file: // src exists and target is a regular file:
srcFile.createNewFile(); Verify.createNewFile(srcFile);
assertTrue(srcFile.exists()); Verify.exists(srcFile);
FileUtil.replaceFile(srcFile, targetFile); FileUtil.replaceFile(srcFile, targetFile);
assertTrue(!srcFile.exists()); Verify.notExists(srcFile);
assertTrue(targetFile.exists()); Verify.exists(targetFile);
// src exists, and target is a non-empty directory: // src exists, and target is a non-empty directory:
srcFile.createNewFile(); Verify.createNewFile(srcFile);
assertTrue(srcFile.exists()); Verify.exists(srcFile);
targetFile.delete(); Verify.delete(targetFile);
targetFile.mkdirs(); Verify.mkdirs(targetFile);
File obstacle = new File(targetFile, "obstacle"); File obstacle = Verify.createNewFile(new File(targetFile, "obstacle"));
obstacle.createNewFile();
assertTrue(obstacle.exists());
assertTrue(targetFile.exists() && targetFile.isDirectory()); assertTrue(targetFile.exists() && targetFile.isDirectory());
try { try {
FileUtil.replaceFile(srcFile, targetFile); FileUtil.replaceFile(srcFile, targetFile);
@ -682,9 +749,9 @@ public class TestFileUtil {
// okay // okay
} }
// check up the post-condition: nothing is deleted: // check up the post-condition: nothing is deleted:
assertTrue(srcFile.exists()); Verify.exists(srcFile);
assertTrue(targetFile.exists() && targetFile.isDirectory()); assertTrue(targetFile.exists() && targetFile.isDirectory());
assertTrue(obstacle.exists()); Verify.exists(obstacle);
} }
@Test (timeout = 30000) @Test (timeout = 30000)
@ -697,13 +764,13 @@ public class TestFileUtil {
assertTrue(tmp1.exists() && tmp2.exists()); assertTrue(tmp1.exists() && tmp2.exists());
assertTrue(tmp1.canWrite() && tmp2.canWrite()); assertTrue(tmp1.canWrite() && tmp2.canWrite());
assertTrue(tmp1.canRead() && tmp2.canRead()); assertTrue(tmp1.canRead() && tmp2.canRead());
tmp1.delete(); Verify.delete(tmp1);
tmp2.delete(); Verify.delete(tmp2);
assertTrue(!tmp1.exists() && !tmp2.exists()); assertTrue(!tmp1.exists() && !tmp2.exists());
} }
@Test (timeout = 30000) @Test (timeout = 30000)
public void testUnZip() throws IOException { public void testUnZip() throws Exception {
// make sa simple zip // make sa simple zip
final File simpleZip = new File(del, FILE); final File simpleZip = new File(del, FILE);
OutputStream os = new FileOutputStream(simpleZip); OutputStream os = new FileOutputStream(simpleZip);
@ -724,18 +791,12 @@ public class TestFileUtil {
// successfully unzip it into an existing dir: // successfully unzip it into an existing dir:
FileUtil.unZip(simpleZip, tmp); FileUtil.unZip(simpleZip, tmp);
// check result: // check result:
assertTrue(new File(tmp, "foo").exists()); Verify.exists(new File(tmp, "foo"));
assertEquals(12, new File(tmp, "foo").length()); assertEquals(12, new File(tmp, "foo").length());
final File regularFile = new File(tmp, "QuickBrownFoxJumpsOverTheLazyDog"); final File regularFile =
regularFile.createNewFile(); Verify.createNewFile(new File(tmp, "QuickBrownFoxJumpsOverTheLazyDog"));
assertTrue(regularFile.exists()); LambdaTestUtils.intercept(IOException.class, () -> FileUtil.unZip(simpleZip, regularFile));
try {
FileUtil.unZip(simpleZip, regularFile);
assertTrue("An IOException expected.", false);
} catch (IOException ioe) {
// okay
}
} }
@Test (timeout = 30000) @Test (timeout = 30000)
@ -781,24 +842,24 @@ public class TestFileUtil {
final File dest = new File(del, "dest"); final File dest = new File(del, "dest");
boolean result = FileUtil.copy(fs, srcPath, dest, false, conf); boolean result = FileUtil.copy(fs, srcPath, dest, false, conf);
assertTrue(result); assertTrue(result);
assertTrue(dest.exists()); Verify.exists(dest);
assertEquals(content.getBytes().length assertEquals(content.getBytes().length
+ System.getProperty("line.separator").getBytes().length, dest.length()); + System.getProperty("line.separator").getBytes().length, dest.length());
assertTrue(srcFile.exists()); // should not be deleted Verify.exists(srcFile); // should not be deleted
// copy regular file, delete src: // copy regular file, delete src:
dest.delete(); Verify.delete(dest);
assertTrue(!dest.exists()); Verify.notExists(dest);
result = FileUtil.copy(fs, srcPath, dest, true, conf); result = FileUtil.copy(fs, srcPath, dest, true, conf);
assertTrue(result); assertTrue(result);
assertTrue(dest.exists()); Verify.exists(dest);
assertEquals(content.getBytes().length assertEquals(content.getBytes().length
+ System.getProperty("line.separator").getBytes().length, dest.length()); + System.getProperty("line.separator").getBytes().length, dest.length());
assertTrue(!srcFile.exists()); // should be deleted Verify.notExists(srcFile); // should be deleted
// copy a dir: // copy a dir:
dest.delete(); Verify.delete(dest);
assertTrue(!dest.exists()); Verify.notExists(dest);
srcPath = new Path(partitioned.toURI()); srcPath = new Path(partitioned.toURI());
result = FileUtil.copy(fs, srcPath, dest, true, conf); result = FileUtil.copy(fs, srcPath, dest, true, conf);
assertTrue(result); assertTrue(result);
@ -810,7 +871,7 @@ public class TestFileUtil {
assertEquals(3 assertEquals(3
+ System.getProperty("line.separator").getBytes().length, f.length()); + System.getProperty("line.separator").getBytes().length, f.length());
} }
assertTrue(!partitioned.exists()); // should be deleted Verify.notExists(partitioned); // should be deleted
} }
@Test (timeout = 30000) @Test (timeout = 30000)
@ -898,8 +959,8 @@ public class TestFileUtil {
// create the symlink // create the symlink
FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath()); FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath());
Assert.assertTrue(file.exists()); Verify.exists(file);
Assert.assertTrue(link.exists()); Verify.exists(link);
File link2 = new File(del, "_link2"); File link2 = new File(del, "_link2");
@ -909,10 +970,10 @@ public class TestFileUtil {
// Make sure the file still exists // Make sure the file still exists
// (NOTE: this would fail on Java6 on Windows if we didn't // (NOTE: this would fail on Java6 on Windows if we didn't
// copy the file in FileUtil#symlink) // copy the file in FileUtil#symlink)
Assert.assertTrue(file.exists()); Verify.exists(file);
Assert.assertTrue(link2.exists()); Verify.exists(link2);
Assert.assertFalse(link.exists()); Verify.notExists(link);
} }
/** /**
@ -927,13 +988,13 @@ public class TestFileUtil {
// create the symlink // create the symlink
FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath()); FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath());
Assert.assertTrue(file.exists()); Verify.exists(file);
Assert.assertTrue(link.exists()); Verify.exists(link);
// make sure that deleting a symlink works properly // make sure that deleting a symlink works properly
Assert.assertTrue(link.delete()); Verify.delete(link);
Assert.assertFalse(link.exists()); Verify.notExists(link);
Assert.assertTrue(file.exists()); Verify.exists(file);
} }
/** /**
@ -960,13 +1021,13 @@ public class TestFileUtil {
Assert.assertEquals(data.length, file.length()); Assert.assertEquals(data.length, file.length());
Assert.assertEquals(data.length, link.length()); Assert.assertEquals(data.length, link.length());
file.delete(); Verify.delete(file);
Assert.assertFalse(file.exists()); Verify.notExists(file);
Assert.assertEquals(0, link.length()); Assert.assertEquals(0, link.length());
link.delete(); Verify.delete(link);
Assert.assertFalse(link.exists()); Verify.notExists(link);
} }
/** /**
@ -1032,7 +1093,7 @@ public class TestFileUtil {
public void testSymlinkSameFile() throws IOException { public void testSymlinkSameFile() throws IOException {
File file = new File(del, FILE); File file = new File(del, FILE);
file.delete(); Verify.delete(file);
// Create a symbolic link // Create a symbolic link
// The operation should succeed // The operation should succeed
@ -1105,21 +1166,21 @@ public class TestFileUtil {
String parentDir = untarDir.getCanonicalPath() + Path.SEPARATOR + "name"; String parentDir = untarDir.getCanonicalPath() + Path.SEPARATOR + "name";
File testFile = new File(parentDir + Path.SEPARATOR + "version"); File testFile = new File(parentDir + Path.SEPARATOR + "version");
Assert.assertTrue(testFile.exists()); Verify.exists(testFile);
Assert.assertTrue(testFile.length() == 0); Assert.assertTrue(testFile.length() == 0);
String imageDir = parentDir + Path.SEPARATOR + "image"; String imageDir = parentDir + Path.SEPARATOR + "image";
testFile = new File(imageDir + Path.SEPARATOR + "fsimage"); testFile = new File(imageDir + Path.SEPARATOR + "fsimage");
Assert.assertTrue(testFile.exists()); Verify.exists(testFile);
Assert.assertTrue(testFile.length() == 157); Assert.assertTrue(testFile.length() == 157);
String currentDir = parentDir + Path.SEPARATOR + "current"; String currentDir = parentDir + Path.SEPARATOR + "current";
testFile = new File(currentDir + Path.SEPARATOR + "fsimage"); testFile = new File(currentDir + Path.SEPARATOR + "fsimage");
Assert.assertTrue(testFile.exists()); Verify.exists(testFile);
Assert.assertTrue(testFile.length() == 4331); Assert.assertTrue(testFile.length() == 4331);
testFile = new File(currentDir + Path.SEPARATOR + "edits"); testFile = new File(currentDir + Path.SEPARATOR + "edits");
Assert.assertTrue(testFile.exists()); Verify.exists(testFile);
Assert.assertTrue(testFile.length() == 1033); Assert.assertTrue(testFile.length() == 1033);
testFile = new File(currentDir + Path.SEPARATOR + "fstime"); testFile = new File(currentDir + Path.SEPARATOR + "fstime");
Assert.assertTrue(testFile.exists()); Verify.exists(testFile);
Assert.assertTrue(testFile.length() == 8); Assert.assertTrue(testFile.length() == 8);
} }
@ -1180,9 +1241,9 @@ public class TestFileUtil {
} }
// create non-jar files, which we expect to not be included in the classpath // create non-jar files, which we expect to not be included in the classpath
Assert.assertTrue(new File(tmp, "text.txt").createNewFile()); Verify.createNewFile(new File(tmp, "text.txt"));
Assert.assertTrue(new File(tmp, "executable.exe").createNewFile()); Verify.createNewFile(new File(tmp, "executable.exe"));
Assert.assertTrue(new File(tmp, "README").createNewFile()); Verify.createNewFile(new File(tmp, "README"));
// create classpath jar // create classpath jar
String wildcardPath = tmp.getCanonicalPath() + File.separator + "*"; String wildcardPath = tmp.getCanonicalPath() + File.separator + "*";
@ -1268,9 +1329,9 @@ public class TestFileUtil {
} }
// create non-jar files, which we expect to not be included in the result // create non-jar files, which we expect to not be included in the result
assertTrue(new File(tmp, "text.txt").createNewFile()); Verify.createNewFile(new File(tmp, "text.txt"));
assertTrue(new File(tmp, "executable.exe").createNewFile()); Verify.createNewFile(new File(tmp, "executable.exe"));
assertTrue(new File(tmp, "README").createNewFile()); Verify.createNewFile(new File(tmp, "README"));
// pass in the directory // pass in the directory
String directory = tmp.getCanonicalPath(); String directory = tmp.getCanonicalPath();
@ -1304,7 +1365,7 @@ public class TestFileUtil {
uri4 = new URI(uris4); uri4 = new URI(uris4);
uri5 = new URI(uris5); uri5 = new URI(uris5);
uri6 = new URI(uris6); uri6 = new URI(uris6);
} catch (URISyntaxException use) { } catch (URISyntaxException ignored) {
} }
// Set up InetAddress // Set up InetAddress
inet1 = mock(InetAddress.class); inet1 = mock(InetAddress.class);
@ -1327,7 +1388,7 @@ public class TestFileUtil {
when(InetAddress.getByName(uris3)).thenReturn(inet3); when(InetAddress.getByName(uris3)).thenReturn(inet3);
when(InetAddress.getByName(uris4)).thenReturn(inet4); when(InetAddress.getByName(uris4)).thenReturn(inet4);
when(InetAddress.getByName(uris5)).thenReturn(inet5); when(InetAddress.getByName(uris5)).thenReturn(inet5);
} catch (UnknownHostException ue) { } catch (UnknownHostException ignored) {
} }
fs1 = mock(FileSystem.class); fs1 = mock(FileSystem.class);
@ -1347,62 +1408,87 @@ public class TestFileUtil {
@Test @Test
public void testCompareFsNull() throws Exception { public void testCompareFsNull() throws Exception {
setupCompareFs(); setupCompareFs();
assertEquals(FileUtil.compareFs(null,fs1),false); assertFalse(FileUtil.compareFs(null, fs1));
assertEquals(FileUtil.compareFs(fs1,null),false); assertFalse(FileUtil.compareFs(fs1, null));
} }
@Test @Test
public void testCompareFsDirectories() throws Exception { public void testCompareFsDirectories() throws Exception {
setupCompareFs(); setupCompareFs();
assertEquals(FileUtil.compareFs(fs1,fs1),true); assertTrue(FileUtil.compareFs(fs1, fs1));
assertEquals(FileUtil.compareFs(fs1,fs2),false); assertFalse(FileUtil.compareFs(fs1, fs2));
assertEquals(FileUtil.compareFs(fs1,fs5),false); assertFalse(FileUtil.compareFs(fs1, fs5));
assertEquals(FileUtil.compareFs(fs3,fs4),true); assertTrue(FileUtil.compareFs(fs3, fs4));
assertEquals(FileUtil.compareFs(fs1,fs6),false); assertFalse(FileUtil.compareFs(fs1, fs6));
} }
@Test(timeout = 8000) @Test(timeout = 8000)
public void testCreateSymbolicLinkUsingJava() throws IOException { public void testCreateSymbolicLinkUsingJava() throws IOException {
final File simpleTar = new File(del, FILE); final File simpleTar = new File(del, FILE);
OutputStream os = new FileOutputStream(simpleTar); OutputStream os = new FileOutputStream(simpleTar);
TarArchiveOutputStream tos = new TarArchiveOutputStream(os); try (TarArchiveOutputStream tos = new TarArchiveOutputStream(os)) {
File untarFile = null;
try {
// Files to tar // Files to tar
final String tmpDir = "tmp/test"; final String tmpDir = "tmp/test";
File tmpDir1 = new File(tmpDir, "dir1/"); File tmpDir1 = new File(tmpDir, "dir1/");
File tmpDir2 = new File(tmpDir, "dir2/"); File tmpDir2 = new File(tmpDir, "dir2/");
// Delete the directories if they already exist Verify.mkdirs(tmpDir1);
tmpDir1.mkdirs(); Verify.mkdirs(tmpDir2);
tmpDir2.mkdirs();
java.nio.file.Path symLink = FileSystems java.nio.file.Path symLink = Paths.get(tmpDir1.getPath(), "sl");
.getDefault().getPath(tmpDir1.getPath() + "/sl");
// Create Symbolic Link // Create Symbolic Link
Files.createSymbolicLink(symLink, Files.createSymbolicLink(symLink, Paths.get(tmpDir2.getPath()));
FileSystems.getDefault().getPath(tmpDir2.getPath())).toString();
assertTrue(Files.isSymbolicLink(symLink.toAbsolutePath())); assertTrue(Files.isSymbolicLink(symLink.toAbsolutePath()));
// put entries in tar file // Put entries in tar file
putEntriesInTar(tos, tmpDir1.getParentFile()); putEntriesInTar(tos, tmpDir1.getParentFile());
tos.close(); tos.close();
untarFile = new File(tmpDir, "2"); File untarFile = new File(tmpDir, "2");
// Untar using java // Untar using Java
FileUtil.unTarUsingJava(simpleTar, untarFile, false); FileUtil.unTarUsingJava(simpleTar, untarFile, false);
// Check symbolic link and other directories are there in untar file // Check symbolic link and other directories are there in untar file
assertTrue(Files.exists(untarFile.toPath())); assertTrue(Files.exists(untarFile.toPath()));
assertTrue(Files.exists(FileSystems.getDefault().getPath(untarFile assertTrue(Files.exists(Paths.get(untarFile.getPath(), tmpDir)));
.getPath(), tmpDir))); assertTrue(Files.isSymbolicLink(Paths.get(untarFile.getPath(), symLink.toString())));
assertTrue(Files.isSymbolicLink(FileSystems.getDefault().getPath(untarFile
.getPath().toString(), symLink.toString())));
} finally { } finally {
FileUtils.deleteDirectory(new File("tmp")); FileUtils.deleteDirectory(new File("tmp"));
tos.close();
} }
}
@Test(expected = IOException.class)
public void testCreateArbitrarySymlinkUsingJava() throws IOException {
final File simpleTar = new File(del, FILE);
OutputStream os = new FileOutputStream(simpleTar);
File rootDir = new File("tmp");
try (TarArchiveOutputStream tos = new TarArchiveOutputStream(os)) {
tos.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
// Create arbitrary dir
File arbitraryDir = new File(rootDir, "arbitrary-dir/");
Verify.mkdirs(arbitraryDir);
// We will tar from the tar-root lineage
File tarRoot = new File(rootDir, "tar-root/");
File symlinkRoot = new File(tarRoot, "dir1/");
Verify.mkdirs(symlinkRoot);
// Create Symbolic Link to an arbitrary dir
java.nio.file.Path symLink = Paths.get(symlinkRoot.getPath(), "sl");
Files.createSymbolicLink(symLink, arbitraryDir.toPath().toAbsolutePath());
// Put entries in tar file
putEntriesInTar(tos, tarRoot);
putEntriesInTar(tos, new File(symLink.toFile(), "dir-outside-tar-root/"));
tos.close();
// Untar using Java
File untarFile = new File(rootDir, "extracted");
FileUtil.unTarUsingJava(simpleTar, untarFile, false);
} finally {
FileUtils.deleteDirectory(rootDir);
}
} }
private void putEntriesInTar(TarArchiveOutputStream tos, File f) private void putEntriesInTar(TarArchiveOutputStream tos, File f)
@ -1496,7 +1582,7 @@ public class TestFileUtil {
String result = FileUtil.readLink(file); String result = FileUtil.readLink(file);
Assert.assertEquals("", result); Assert.assertEquals("", result);
file.delete(); Verify.delete(file);
} }
/** /**