BAEL-3462 Univosity Parsers (#9222)
* BAEL-3462 CSV Univosity Example * Added a row processor example
This commit is contained in:
Amy DeGregorio
GitHub
parent
832b32210c
commit
c66c60bcff
|
|
@ -116,6 +116,11 @@
|
|||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<version>${slf4j.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.univocity</groupId>
|
||||
<artifactId>univocity-parsers</artifactId>
|
||||
<version>${univocity.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.awaitility</groupId>
|
||||
<artifactId>awaitility</artifactId>
|
||||
|
|
@ -175,6 +180,7 @@
|
|||
<assertj.version>3.6.2</assertj.version>
|
||||
<slf4j.version>1.7.25</slf4j.version>
|
||||
<awaitility.version>3.0.0</awaitility.version>
|
||||
<univocity.version>2.8.4</univocity.version>
|
||||
<renjin.version>RELEASE</renjin.version>
|
||||
<rcaller.version>3.0</rcaller.version>
|
||||
<rserve.version>1.8.1</rserve.version>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,80 @@
|
|||
package com.baeldung.univocity;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.Writer;
|
||||
import java.util.List;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.baeldung.univocity.model.Product;
|
||||
import com.univocity.parsers.common.processor.BeanWriterProcessor;
|
||||
import com.univocity.parsers.csv.CsvWriter;
|
||||
import com.univocity.parsers.csv.CsvWriterSettings;
|
||||
import com.univocity.parsers.fixed.FixedWidthFields;
|
||||
import com.univocity.parsers.fixed.FixedWidthWriter;
|
||||
import com.univocity.parsers.fixed.FixedWidthWriterSettings;
|
||||
import com.univocity.parsers.tsv.TsvWriter;
|
||||
import com.univocity.parsers.tsv.TsvWriterSettings;
|
||||
|
||||
public class OutputService {
|
||||
private Logger logger = LoggerFactory.getLogger(ParsingService.class);
|
||||
|
||||
public enum OutputType {
|
||||
CSV, TSV, FIXED_WIDTH
|
||||
};
|
||||
|
||||
public boolean writeData(List<Object[]> products, OutputType outputType, String outputPath) {
|
||||
try (Writer outputWriter = new OutputStreamWriter(new FileOutputStream(new File(outputPath)), "UTF-8")) {
|
||||
switch (outputType) {
|
||||
case CSV: {
|
||||
CsvWriter writer = new CsvWriter(outputWriter, new CsvWriterSettings());
|
||||
writer.writeRowsAndClose(products);
|
||||
}
|
||||
break;
|
||||
case TSV: {
|
||||
TsvWriter writer = new TsvWriter(outputWriter, new TsvWriterSettings());
|
||||
writer.writeRowsAndClose(products);
|
||||
}
|
||||
break;
|
||||
case FIXED_WIDTH: {
|
||||
FixedWidthFields fieldLengths = new FixedWidthFields(8, 30, 10);
|
||||
FixedWidthWriterSettings settings = new FixedWidthWriterSettings(fieldLengths);
|
||||
FixedWidthWriter writer = new FixedWidthWriter(outputWriter, settings);
|
||||
writer.writeRowsAndClose(products);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
logger.warn("Invalid OutputType: " + outputType);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch (IOException e) {
|
||||
logger.error(e.getMessage());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean writeBeanToFixedWidthFile(List<Product> products, String outputPath) {
|
||||
try (Writer outputWriter = new OutputStreamWriter(new FileOutputStream(new File(outputPath)), "UTF-8")) {
|
||||
BeanWriterProcessor<Product> rowProcessor = new BeanWriterProcessor<Product>(Product.class);
|
||||
FixedWidthFields fieldLengths = new FixedWidthFields(8, 30, 10);
|
||||
FixedWidthWriterSettings settings = new FixedWidthWriterSettings(fieldLengths);
|
||||
settings.setHeaders("product_no", "description", "unit_price");
|
||||
settings.setRowWriterProcessor(rowProcessor);
|
||||
FixedWidthWriter writer = new FixedWidthWriter(outputWriter, settings);
|
||||
writer.writeHeaders();
|
||||
for (Product product : products) {
|
||||
writer.processRecord(product);
|
||||
}
|
||||
writer.close();
|
||||
return true;
|
||||
} catch (IOException e) {
|
||||
logger.error(e.getMessage());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
package com.baeldung.univocity;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.Reader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.baeldung.univocity.model.Product;
|
||||
import com.univocity.parsers.common.processor.BatchedColumnProcessor;
|
||||
import com.univocity.parsers.common.processor.BeanListProcessor;
|
||||
import com.univocity.parsers.csv.CsvParser;
|
||||
import com.univocity.parsers.csv.CsvParserSettings;
|
||||
import com.univocity.parsers.fixed.FixedWidthFields;
|
||||
import com.univocity.parsers.fixed.FixedWidthParser;
|
||||
import com.univocity.parsers.fixed.FixedWidthParserSettings;
|
||||
|
||||
public class ParsingService {
|
||||
private Logger logger = LoggerFactory.getLogger(ParsingService.class);
|
||||
|
||||
public List<String[]> parseCsvFile(String relativePath) {
|
||||
try (Reader inputReader = new InputStreamReader(new FileInputStream(new File(relativePath)), "UTF-8")) {
|
||||
CsvParserSettings settings = new CsvParserSettings();
|
||||
settings.setMaxCharsPerColumn(100);
|
||||
settings.setMaxColumns(50);
|
||||
CsvParser parser = new CsvParser(settings);
|
||||
List<String[]> parsedRows = parser.parseAll(inputReader);
|
||||
return parsedRows;
|
||||
} catch (IOException e) {
|
||||
logger.error("IOException opening file: " + relativePath + " " + e.getMessage());
|
||||
return new ArrayList<String[]>();
|
||||
}
|
||||
}
|
||||
|
||||
public List<String[]> parseFixedWidthFile(String relativePath) {
|
||||
try (Reader inputReader = new InputStreamReader(new FileInputStream(new File(relativePath)), "UTF-8")) {
|
||||
FixedWidthFields fieldLengths = new FixedWidthFields(8, 30, 10);
|
||||
FixedWidthParserSettings settings = new FixedWidthParserSettings(fieldLengths);
|
||||
|
||||
FixedWidthParser parser = new FixedWidthParser(settings);
|
||||
List<String[]> parsedRows = parser.parseAll(inputReader);
|
||||
return parsedRows;
|
||||
} catch (IOException e) {
|
||||
logger.error("IOException opening file: " + relativePath + " " + e.getMessage());
|
||||
return new ArrayList<String[]>();
|
||||
}
|
||||
}
|
||||
|
||||
public List<Product> parseCsvFileIntoBeans(String relativePath) {
|
||||
try (Reader inputReader = new InputStreamReader(new FileInputStream(new File(relativePath)), "UTF-8")) {
|
||||
BeanListProcessor<Product> rowProcessor = new BeanListProcessor<Product>(Product.class);
|
||||
CsvParserSettings settings = new CsvParserSettings();
|
||||
settings.setHeaderExtractionEnabled(true);
|
||||
settings.setProcessor(rowProcessor);
|
||||
CsvParser parser = new CsvParser(settings);
|
||||
parser.parse(inputReader);
|
||||
return rowProcessor.getBeans();
|
||||
} catch (IOException e) {
|
||||
logger.error("IOException opening file: " + relativePath + " " + e.getMessage());
|
||||
return new ArrayList<Product>();
|
||||
}
|
||||
}
|
||||
|
||||
public List<String[]> parseCsvFileInBatches(String relativePath) {
|
||||
try (Reader inputReader = new InputStreamReader(new FileInputStream(new File(relativePath)), "UTF-8")) {
|
||||
CsvParserSettings settings = new CsvParserSettings();
|
||||
settings.setProcessor(new BatchedColumnProcessor(5) {
|
||||
@Override
|
||||
public void batchProcessed(int rowsInThisBatch) {
|
||||
}
|
||||
});
|
||||
CsvParser parser = new CsvParser(settings);
|
||||
List<String[]> parsedRows = parser.parseAll(inputReader);
|
||||
return parsedRows;
|
||||
} catch (IOException e) {
|
||||
logger.error("IOException opening file: " + relativePath + " " + e.getMessage());
|
||||
return new ArrayList<String[]>();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
package com.baeldung.univocity.model;
|
||||
|
||||
import com.univocity.parsers.annotations.Parsed;
|
||||
|
||||
public class Product {
|
||||
|
||||
@Parsed(field = "product_no")
|
||||
private String productNumber;
|
||||
|
||||
@Parsed
|
||||
private String description;
|
||||
|
||||
@Parsed(field = "unit_price")
|
||||
private float unitPrice;
|
||||
|
||||
public String getProductNumber() {
|
||||
return productNumber;
|
||||
}
|
||||
|
||||
public void setProductNumber(String productNumber) {
|
||||
this.productNumber = productNumber;
|
||||
}
|
||||
|
||||
public String getDescription() {
|
||||
return description;
|
||||
}
|
||||
|
||||
public void setDescription(String description) {
|
||||
this.description = description;
|
||||
}
|
||||
|
||||
public float getUnitPrice() {
|
||||
return unitPrice;
|
||||
}
|
||||
|
||||
public void setUnitPrice(float unitPrice) {
|
||||
this.unitPrice = unitPrice;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Product [Product Number: " + productNumber + ", Description: " + description + ", Unit Price: " + unitPrice + "]";
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
package com.baeldung.univocity;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import com.baeldung.univocity.model.Product;
|
||||
|
||||
public class ParsingServiceUnitTest {
|
||||
|
||||
@Test
|
||||
public void givenCsvFile_thenParsedResultsShouldBeReturned() {
|
||||
ParsingService parsingService = new ParsingService();
|
||||
List<String[]> productData = parsingService.parseCsvFile("src/test/resources/productList.csv");
|
||||
assertEquals(3, productData.size());
|
||||
assertEquals(3, productData.get(0).length);
|
||||
assertEquals("A8993-10", productData.get(0)[0]);
|
||||
assertEquals("Extra large widget", productData.get(0)[1]);
|
||||
assertEquals("35.42", productData.get(0)[2]);
|
||||
assertEquals("D-2938-1", productData.get(1)[0]);
|
||||
assertEquals("Winding widget \"Deluxe Model\"", productData.get(1)[1]);
|
||||
assertEquals("245.99", productData.get(1)[2]);
|
||||
assertEquals("R3212-32", productData.get(2)[0]);
|
||||
assertEquals("Standard widget", productData.get(2)[1]);
|
||||
assertEquals("2.34", productData.get(2)[2]);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void givenFixedWidthFile_thenParsedResultsShouldBeReturned() {
|
||||
ParsingService parsingService = new ParsingService();
|
||||
List<String[]> productData = parsingService.parseFixedWidthFile("src/test/resources/productList.txt");
|
||||
// Note: any extra spaces on the end will cause a null line to be added
|
||||
assertEquals(3, productData.size());
|
||||
assertEquals(3, productData.get(0).length);
|
||||
assertEquals("A8993-10", productData.get(0)[0]);
|
||||
assertEquals("Extra large widget", productData.get(0)[1]);
|
||||
assertEquals("35.42", productData.get(0)[2]);
|
||||
assertEquals("D-2938-1", productData.get(1)[0]);
|
||||
assertEquals("Winding widget \"Deluxe Model\"", productData.get(1)[1]);
|
||||
assertEquals("245.99", productData.get(1)[2]);
|
||||
assertEquals("R3212-32", productData.get(2)[0]);
|
||||
assertEquals("Standard widget", productData.get(2)[1]);
|
||||
assertEquals("2.34", productData.get(2)[2]);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void givenDataAndCsvOutputType_thenCsvFileProduced() {
|
||||
OutputService outputService = new OutputService();
|
||||
List<Object[]> productData = new ArrayList<>();
|
||||
productData.add(new Object[] { "1000-3-0", "Widget No. 96", "5.67" });
|
||||
productData.add(new Object[] { "G930-M-P", "1/4\" Wocket", ".67" });
|
||||
productData.add(new Object[] { "8080-0-M", "No. 54 Jumbo Widget", "35.74" });
|
||||
outputService.writeData(productData, OutputService.OutputType.CSV, "src/test/resources/outputProductList.csv");
|
||||
|
||||
ParsingService parsingService = new ParsingService();
|
||||
List<String[]> writtenData = parsingService.parseCsvFile("src/test/resources/outputProductList.csv");
|
||||
assertEquals(3, writtenData.size());
|
||||
assertEquals(3, writtenData.get(0).length);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void givenDataAndFixedWidthOutputType_thenFixedWidthFileProduced() {
|
||||
OutputService outputService = new OutputService();
|
||||
List<Object[]> productData = new ArrayList<>();
|
||||
productData.add(new Object[] { "1000-3-0", "Widget No. 96", "5.67" });
|
||||
productData.add(new Object[] { "G930-M-P", "1/4\" Wocket", ".67" });
|
||||
productData.add(new Object[] { "8080-0-M", "No. 54 Jumbo Widget", "35.74" });
|
||||
outputService.writeData(productData, OutputService.OutputType.FIXED_WIDTH, "src/test/resources/outputProductList.txt");
|
||||
|
||||
ParsingService parsingService = new ParsingService();
|
||||
List<String[]> writtenData = parsingService.parseFixedWidthFile("src/test/resources/outputProductList.txt");
|
||||
assertEquals(3, writtenData.size());
|
||||
assertEquals(3, writtenData.get(0).length);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void givenCsvFile_thenCsvFileParsedIntoBeans() {
|
||||
ParsingService parsingService = new ParsingService();
|
||||
List<Product> products = parsingService.parseCsvFileIntoBeans("src/test/resources/productListWithHeaders.csv");
|
||||
assertEquals(2, products.size());
|
||||
assertEquals("Product [Product Number: 99-378AG, Description: Wocket Widget #42, Unit Price: 3.56]", products.get(0)
|
||||
.toString());
|
||||
assertEquals("Product [Product Number: TB-333-0, Description: Turbine Widget replacement kit, Unit Price: 800.99]", products.get(1)
|
||||
.toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void givenLisOfProduct_thenWriteFixedWidthFile() {
|
||||
OutputService outputService = new OutputService();
|
||||
Product product = new Product();
|
||||
product.setProductNumber("007-PPG0");
|
||||
product.setDescription("3/8\" x 1\" Wocket");
|
||||
product.setUnitPrice(45.99f);
|
||||
List<Product> products = new ArrayList<>();
|
||||
products.add(product);
|
||||
outputService.writeBeanToFixedWidthFile(products, "src/test/resources/productListWithHeaders.txt");
|
||||
ParsingService parsingService = new ParsingService();
|
||||
List<String[]> writtenData = parsingService.parseFixedWidthFile("src/test/resources/productListWithHeaders.txt");
|
||||
assertEquals(2, writtenData.size());
|
||||
assertEquals(3, writtenData.get(0).length);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void givenLargeCsvFile_thenParsedDataShouldBeReturned() {
|
||||
ParsingService parsingService = new ParsingService();
|
||||
List<String[]> productData = parsingService.parseCsvFileInBatches("src/test/resources/largeProductList.csv");
|
||||
assertEquals(36, productData.size());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
A8993-10,"Extra large widget",35.42
|
||||
D-2938-1,"Winding widget ""Deluxe Model""",245.99
|
||||
R3212-32,"Standard widget",2.34
|
||||
A8993-10,"Extra large widget",35.42
|
||||
D-2938-1,"Winding widget ""Deluxe Model""",245.99
|
||||
R3212-32,"Standard widget",2.34
|
||||
A8993-10,"Extra large widget",35.42
|
||||
D-2938-1,"Winding widget ""Deluxe Model""",245.99
|
||||
R3212-32,"Standard widget",2.34
|
||||
A8993-10,"Extra large widget",35.42
|
||||
D-2938-1,"Winding widget ""Deluxe Model""",245.99
|
||||
R3212-32,"Standard widget",2.34
|
||||
A8993-10,"Extra large widget",35.42
|
||||
D-2938-1,"Winding widget ""Deluxe Model""",245.99
|
||||
R3212-32,"Standard widget",2.34
|
||||
A8993-10,"Extra large widget",35.42
|
||||
D-2938-1,"Winding widget ""Deluxe Model""",245.99
|
||||
R3212-32,"Standard widget",2.34
|
||||
A8993-10,"Extra large widget",35.42
|
||||
D-2938-1,"Winding widget ""Deluxe Model""",245.99
|
||||
R3212-32,"Standard widget",2.34
|
||||
A8993-10,"Extra large widget",35.42
|
||||
D-2938-1,"Winding widget ""Deluxe Model""",245.99
|
||||
R3212-32,"Standard widget",2.34
|
||||
A8993-10,"Extra large widget",35.42
|
||||
D-2938-1,"Winding widget ""Deluxe Model""",245.99
|
||||
R3212-32,"Standard widget",2.34
|
||||
A8993-10,"Extra large widget",35.42
|
||||
D-2938-1,"Winding widget ""Deluxe Model""",245.99
|
||||
R3212-32,"Standard widget",2.34
|
||||
A8993-10,"Extra large widget",35.42
|
||||
D-2938-1,"Winding widget ""Deluxe Model""",245.99
|
||||
R3212-32,"Standard widget",2.34
|
||||
A8993-10,"Extra large widget",35.42
|
||||
D-2938-1,"Winding widget ""Deluxe Model""",245.99
|
||||
R3212-32,"Standard widget",2.34
|
||||
|
|
|
@ -0,0 +1,3 @@
|
|||
A8993-10,"Extra large widget",35.42
|
||||
D-2938-1,"Winding widget ""Deluxe Model""",245.99
|
||||
R3212-32,"Standard widget",2.34
|
||||
|
|
|
@ -0,0 +1,3 @@
|
|||
A8993-10Extra large widget 35.42
|
||||
D-2938-1Winding widget "Deluxe Model" 245.99
|
||||
R3212-32Standard widget 2.34
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
product_no, description, unit_price
|
||||
99-378AG,Wocket Widget #42,3.56
|
||||
TB-333-0,Turbine Widget replacement kit,800.99
|
||||
|
|
|
@ -0,0 +1,2 @@
|
|||
product_description unit_price
|
||||
007-PPG03/8" x 1" Wocket 45.99
|
||||
Loading…
Reference in New Issue