Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-16790. Add Write Convenience Methods. 

Contributed by David Mollitor.

This adds operations in FileUtil to write text to a file via
either a FileSystem or FileContext instance.

Change-Id: I5fe8fcf1bdbdbc734e137f922a75a822f2b88410
This commit is contained in:
David Mollitor 2020-01-15 18:38:34 +00:00 committed by Steve Loughran
parent 45b23fd74a
commit 8c150ed484
No known key found for this signature in database
GPG Key ID: D22CF846DBB162A0
2 changed files with 414 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

236
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
View File

@ -21,6 +21,7 @@ package org.apache.hadoop.fs;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
@ -29,10 +30,13 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.net.InetAddress;
import java.net.URI;
import java.net.UnknownHostException;
import java.nio.charset.Charset;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.AccessDeniedException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
@ -40,6 +44,7 @@ import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@ -1647,4 +1652,235 @@ public class FileUtil {
// check for ports
return srcUri.getPort()==dstUri.getPort();
}
/**
* Writes bytes to a file. This utility method opens the file for writing,
* creating the file if it does not exist, or overwrites an existing file. All
* bytes in the byte array are written to the file.
*
* @param fs the file system with which to create the file
* @param path the path to the file
* @param bytes the byte array with the bytes to write
*
* @return the file system
*
* @throws NullPointerException if any of the arguments are {@code null}
* @throws IOException if an I/O error occurs creating or writing to the file
*/
public static FileSystem write(final FileSystem fs, final Path path,
final byte[] bytes) throws IOException {
Objects.requireNonNull(path);
Objects.requireNonNull(bytes);
try (FSDataOutputStream out = fs.createFile(path).overwrite(true).build()) {
out.write(bytes);
}
return fs;
}
/**
* Writes bytes to a file. This utility method opens the file for writing,
* creating the file if it does not exist, or overwrites an existing file. All
* bytes in the byte array are written to the file.
*
* @param fileContext the file context with which to create the file
* @param path the path to the file
* @param bytes the byte array with the bytes to write
*
* @return the file context
*
* @throws NullPointerException if any of the arguments are {@code null}
* @throws IOException if an I/O error occurs creating or writing to the file
*/
public static FileContext write(final FileContext fileContext,
final Path path, final byte[] bytes) throws IOException {
Objects.requireNonNull(path);
Objects.requireNonNull(bytes);
try (FSDataOutputStream out =
fileContext.create(path).overwrite(true).build()) {
out.write(bytes);
}
return fileContext;
}
/**
* Write lines of text to a file. Each line is a char sequence and is written
* to the file in sequence with each line terminated by the platform's line
* separator, as defined by the system property {@code
* line.separator}. Characters are encoded into bytes using the specified
* charset. This utility method opens the file for writing, creating the file
* if it does not exist, or overwrites an existing file.
*
* @param fs the file system with which to create the file
* @param path the path to the file
* @param lines a Collection to iterate over the char sequences
* @param cs the charset to use for encoding
*
* @return the file system
*
* @throws NullPointerException if any of the arguments are {@code null}
* @throws IOException if an I/O error occurs creating or writing to the file
*/
public static FileSystem write(final FileSystem fs, final Path path,
final Iterable<? extends CharSequence> lines, final Charset cs)
throws IOException {
Objects.requireNonNull(path);
Objects.requireNonNull(lines);
Objects.requireNonNull(cs);
CharsetEncoder encoder = cs.newEncoder();
try (FSDataOutputStream out = fs.createFile(path).overwrite(true).build();
BufferedWriter writer =
new BufferedWriter(new OutputStreamWriter(out, encoder))) {
for (CharSequence line : lines) {
writer.append(line);
writer.newLine();
}
}
return fs;
}
/**
* Write lines of text to a file. Each line is a char sequence and is written
* to the file in sequence with each line terminated by the platform's line
* separator, as defined by the system property {@code
* line.separator}. Characters are encoded into bytes using the specified
* charset. This utility method opens the file for writing, creating the file
* if it does not exist, or overwrites an existing file.
*
* @param fileContext the file context with which to create the file
* @param path the path to the file
* @param lines a Collection to iterate over the char sequences
* @param cs the charset to use for encoding
*
* @return the file context
*
* @throws NullPointerException if any of the arguments are {@code null}
* @throws IOException if an I/O error occurs creating or writing to the file
*/
public static FileContext write(final FileContext fileContext,
final Path path, final Iterable<? extends CharSequence> lines,
final Charset cs) throws IOException {
Objects.requireNonNull(path);
Objects.requireNonNull(lines);
Objects.requireNonNull(cs);
CharsetEncoder encoder = cs.newEncoder();
try (FSDataOutputStream out = fileContext.create(path).overwrite(true).build();
BufferedWriter writer =
new BufferedWriter(new OutputStreamWriter(out, encoder))) {
for (CharSequence line : lines) {
writer.append(line);
writer.newLine();
}
}
return fileContext;
}
/**
* Write a line of text to a file. Characters are encoded into bytes using the
* specified charset. This utility method opens the file for writing, creating
* the file if it does not exist, or overwrites an existing file.
*
* @param fs the file system with which to create the file
* @param path the path to the file
* @param charseq the char sequence to write to the file
* @param cs the charset to use for encoding
*
* @return the file system
*
* @throws NullPointerException if any of the arguments are {@code null}
* @throws IOException if an I/O error occurs creating or writing to the file
*/
public static FileSystem write(final FileSystem fs, final Path path,
final CharSequence charseq, final Charset cs) throws IOException {
Objects.requireNonNull(path);
Objects.requireNonNull(charseq);
Objects.requireNonNull(cs);
CharsetEncoder encoder = cs.newEncoder();
try (FSDataOutputStream out = fs.createFile(path).overwrite(true).build();
BufferedWriter writer =
new BufferedWriter(new OutputStreamWriter(out, encoder))) {
writer.append(charseq);
}
return fs;
}
/**
* Write a line of text to a file. Characters are encoded into bytes using the
* specified charset. This utility method opens the file for writing, creating
* the file if it does not exist, or overwrites an existing file.
*
* @param FileContext the file context with which to create the file
* @param path the path to the file
* @param charseq the char sequence to write to the file
* @param cs the charset to use for encoding
*
* @return the file context
*
* @throws NullPointerException if any of the arguments are {@code null}
* @throws IOException if an I/O error occurs creating or writing to the file
*/
public static FileContext write(final FileContext fs, final Path path,
final CharSequence charseq, final Charset cs) throws IOException {
Objects.requireNonNull(path);
Objects.requireNonNull(charseq);
Objects.requireNonNull(cs);
CharsetEncoder encoder = cs.newEncoder();
try (FSDataOutputStream out = fs.create(path).overwrite(true).build();
BufferedWriter writer =
new BufferedWriter(new OutputStreamWriter(out, encoder))) {
writer.append(charseq);
}
return fs;
}
/**
* Write a line of text to a file. Characters are encoded into bytes using
* UTF-8. This utility method opens the file for writing, creating the file if
* it does not exist, or overwrites an existing file.
*
* @param fs the files system with which to create the file
* @param path the path to the file
* @param charseq the char sequence to write to the file
*
* @return the file system
*
* @throws NullPointerException if any of the arguments are {@code null}
* @throws IOException if an I/O error occurs creating or writing to the file
*/
public static FileSystem write(final FileSystem fs, final Path path,
final CharSequence charseq) throws IOException {
return write(fs, path, charseq, StandardCharsets.UTF_8);
}
/**
* Write a line of text to a file. Characters are encoded into bytes using
* UTF-8. This utility method opens the file for writing, creating the file if
* it does not exist, or overwrites an existing file.
*
* @param fileContext the files system with which to create the file
* @param path the path to the file
* @param charseq the char sequence to write to the file
*
* @return the file context
*
* @throws NullPointerException if any of the arguments are {@code null}
* @throws IOException if an I/O error occurs creating or writing to the file
*/
public static FileContext write(final FileContext fileContext,
final Path path, final CharSequence charseq) throws IOException {
return write(fileContext, path, charseq, StandardCharsets.UTF_8);
}
}

178
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.fs;
import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
@ -44,6 +45,7 @@ import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.jar.Attributes;
@ -1493,4 +1495,180 @@ public class TestFileUtil {
file.delete();
}
/**
* Test that bytes are written out correctly to the local file system.
*/
@Test
public void testWriteBytesFileSystem() throws IOException {
setupDirs();
URI uri = tmp.toURI();
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(uri, conf);
Path testPath = new Path(new Path(uri), "writebytes.out");
byte[] write = new byte[] {0x00, 0x01, 0x02, 0x03};
FileUtil.write(fs, testPath, write);
byte[] read = FileUtils.readFileToByteArray(new File(testPath.toUri()));
assertArrayEquals(write, read);
}
/**
* Test that a Collection of Strings are written out correctly to the local
* file system.
*/
@Test
public void testWriteStringsFileSystem() throws IOException {
setupDirs();
URI uri = tmp.toURI();
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(uri, conf);
Path testPath = new Path(new Path(uri), "writestrings.out");
Collection<String> write = Arrays.asList("over", "the", "lazy", "dog");
FileUtil.write(fs, testPath, write, StandardCharsets.UTF_8);
List<String> read =
FileUtils.readLines(new File(testPath.toUri()), StandardCharsets.UTF_8);
assertEquals(write, read);
}
/**
* Test that a String is written out correctly to the local file system.
*/
@Test
public void testWriteStringFileSystem() throws IOException {
setupDirs();
URI uri = tmp.toURI();
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(uri, conf);
Path testPath = new Path(new Path(uri), "writestring.out");
String write = "A" + "\u00ea" + "\u00f1" + "\u00fc" + "C";
FileUtil.write(fs, testPath, write, StandardCharsets.UTF_8);
String read = FileUtils.readFileToString(new File(testPath.toUri()),
StandardCharsets.UTF_8);
assertEquals(write, read);
}
/**
* Test that a String is written out correctly to the local file system
* without specifying a character set.
*/
@Test
public void testWriteStringNoCharSetFileSystem() throws IOException {
setupDirs();
URI uri = tmp.toURI();
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(uri, conf);
Path testPath = new Path(new Path(uri), "writestring.out");
String write = "A" + "\u00ea" + "\u00f1" + "\u00fc" + "C";
FileUtil.write(fs, testPath, write);
String read = FileUtils.readFileToString(new File(testPath.toUri()),
StandardCharsets.UTF_8);
assertEquals(write, read);
}
/**
* Test that bytes are written out correctly to the local file system.
*/
@Test
public void testWriteBytesFileContext() throws IOException {
setupDirs();
URI uri = tmp.toURI();
Configuration conf = new Configuration();
FileContext fc = FileContext.getFileContext(uri, conf);
Path testPath = new Path(new Path(uri), "writebytes.out");
byte[] write = new byte[] {0x00, 0x01, 0x02, 0x03};
FileUtil.write(fc, testPath, write);
byte[] read = FileUtils.readFileToByteArray(new File(testPath.toUri()));
assertArrayEquals(write, read);
}
/**
* Test that a Collection of Strings are written out correctly to the local
* file system.
*/
@Test
public void testWriteStringsFileContext() throws IOException {
setupDirs();
URI uri = tmp.toURI();
Configuration conf = new Configuration();
FileContext fc = FileContext.getFileContext(uri, conf);
Path testPath = new Path(new Path(uri), "writestrings.out");
Collection<String> write = Arrays.asList("over", "the", "lazy", "dog");
FileUtil.write(fc, testPath, write, StandardCharsets.UTF_8);
List<String> read =
FileUtils.readLines(new File(testPath.toUri()), StandardCharsets.UTF_8);
assertEquals(write, read);
}
/**
* Test that a String is written out correctly to the local file system.
*/
@Test
public void testWriteStringFileContext() throws IOException {
setupDirs();
URI uri = tmp.toURI();
Configuration conf = new Configuration();
FileContext fc = FileContext.getFileContext(uri, conf);
Path testPath = new Path(new Path(uri), "writestring.out");
String write = "A" + "\u00ea" + "\u00f1" + "\u00fc" + "C";
FileUtil.write(fc, testPath, write, StandardCharsets.UTF_8);
String read = FileUtils.readFileToString(new File(testPath.toUri()),
StandardCharsets.UTF_8);
assertEquals(write, read);
}
/**
* Test that a String is written out correctly to the local file system
* without specifying a character set.
*/
@Test
public void testWriteStringNoCharSetFileContext() throws IOException {
setupDirs();
URI uri = tmp.toURI();
Configuration conf = new Configuration();
FileContext fc = FileContext.getFileContext(uri, conf);
Path testPath = new Path(new Path(uri), "writestring.out");
String write = "A" + "\u00ea" + "\u00f1" + "\u00fc" + "C";
FileUtil.write(fc, testPath, write);
String read = FileUtils.readFileToString(new File(testPath.toUri()),
StandardCharsets.UTF_8);
assertEquals(write, read);
}
}