diff --git a/libraries-data-2/pom.xml b/libraries-data-2/pom.xml index 73c5452f77..cbb24edd3f 100644 --- a/libraries-data-2/pom.xml +++ b/libraries-data-2/pom.xml @@ -116,6 +116,11 @@ slf4j-log4j12 ${slf4j.version} + + com.univocity + univocity-parsers + ${univocity.version} + org.awaitility awaitility @@ -175,6 +180,7 @@ 3.6.2 1.7.25 3.0.0 + 2.8.4 RELEASE 3.0 1.8.1 diff --git a/libraries-data-2/src/main/java/com/baeldung/univocity/OutputService.java b/libraries-data-2/src/main/java/com/baeldung/univocity/OutputService.java new file mode 100644 index 0000000000..b316a602ad --- /dev/null +++ b/libraries-data-2/src/main/java/com/baeldung/univocity/OutputService.java @@ -0,0 +1,80 @@ +package com.baeldung.univocity; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.baeldung.univocity.model.Product; +import com.univocity.parsers.common.processor.BeanWriterProcessor; +import com.univocity.parsers.csv.CsvWriter; +import com.univocity.parsers.csv.CsvWriterSettings; +import com.univocity.parsers.fixed.FixedWidthFields; +import com.univocity.parsers.fixed.FixedWidthWriter; +import com.univocity.parsers.fixed.FixedWidthWriterSettings; +import com.univocity.parsers.tsv.TsvWriter; +import com.univocity.parsers.tsv.TsvWriterSettings; + +public class OutputService { + private Logger logger = LoggerFactory.getLogger(ParsingService.class); + + public enum OutputType { + CSV, TSV, FIXED_WIDTH + }; + + public boolean writeData(List products, OutputType outputType, String outputPath) { + try (Writer outputWriter = new OutputStreamWriter(new FileOutputStream(new File(outputPath)), "UTF-8")) { + switch (outputType) { + case CSV: { + CsvWriter writer = new CsvWriter(outputWriter, new CsvWriterSettings()); + writer.writeRowsAndClose(products); + } + break; + case TSV: { + TsvWriter writer = new TsvWriter(outputWriter, new TsvWriterSettings()); + writer.writeRowsAndClose(products); + } + break; + case FIXED_WIDTH: { + FixedWidthFields fieldLengths = new FixedWidthFields(8, 30, 10); + FixedWidthWriterSettings settings = new FixedWidthWriterSettings(fieldLengths); + FixedWidthWriter writer = new FixedWidthWriter(outputWriter, settings); + writer.writeRowsAndClose(products); + } + break; + default: + logger.warn("Invalid OutputType: " + outputType); + return false; + } + return true; + } catch (IOException e) { + logger.error(e.getMessage()); + return false; + } + } + + public boolean writeBeanToFixedWidthFile(List products, String outputPath) { + try (Writer outputWriter = new OutputStreamWriter(new FileOutputStream(new File(outputPath)), "UTF-8")) { + BeanWriterProcessor rowProcessor = new BeanWriterProcessor(Product.class); + FixedWidthFields fieldLengths = new FixedWidthFields(8, 30, 10); + FixedWidthWriterSettings settings = new FixedWidthWriterSettings(fieldLengths); + settings.setHeaders("product_no", "description", "unit_price"); + settings.setRowWriterProcessor(rowProcessor); + FixedWidthWriter writer = new FixedWidthWriter(outputWriter, settings); + writer.writeHeaders(); + for (Product product : products) { + writer.processRecord(product); + } + writer.close(); + return true; + } catch (IOException e) { + logger.error(e.getMessage()); + return false; + } + } +} diff --git a/libraries-data-2/src/main/java/com/baeldung/univocity/ParsingService.java b/libraries-data-2/src/main/java/com/baeldung/univocity/ParsingService.java new file mode 100644 index 0000000000..a88023b5d3 --- /dev/null +++ b/libraries-data-2/src/main/java/com/baeldung/univocity/ParsingService.java @@ -0,0 +1,85 @@ +package com.baeldung.univocity; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.baeldung.univocity.model.Product; +import com.univocity.parsers.common.processor.BatchedColumnProcessor; +import com.univocity.parsers.common.processor.BeanListProcessor; +import com.univocity.parsers.csv.CsvParser; +import com.univocity.parsers.csv.CsvParserSettings; +import com.univocity.parsers.fixed.FixedWidthFields; +import com.univocity.parsers.fixed.FixedWidthParser; +import com.univocity.parsers.fixed.FixedWidthParserSettings; + +public class ParsingService { + private Logger logger = LoggerFactory.getLogger(ParsingService.class); + + public List parseCsvFile(String relativePath) { + try (Reader inputReader = new InputStreamReader(new FileInputStream(new File(relativePath)), "UTF-8")) { + CsvParserSettings settings = new CsvParserSettings(); + settings.setMaxCharsPerColumn(100); + settings.setMaxColumns(50); + CsvParser parser = new CsvParser(settings); + List parsedRows = parser.parseAll(inputReader); + return parsedRows; + } catch (IOException e) { + logger.error("IOException opening file: " + relativePath + " " + e.getMessage()); + return new ArrayList(); + } + } + + public List parseFixedWidthFile(String relativePath) { + try (Reader inputReader = new InputStreamReader(new FileInputStream(new File(relativePath)), "UTF-8")) { + FixedWidthFields fieldLengths = new FixedWidthFields(8, 30, 10); + FixedWidthParserSettings settings = new FixedWidthParserSettings(fieldLengths); + + FixedWidthParser parser = new FixedWidthParser(settings); + List parsedRows = parser.parseAll(inputReader); + return parsedRows; + } catch (IOException e) { + logger.error("IOException opening file: " + relativePath + " " + e.getMessage()); + return new ArrayList(); + } + } + + public List parseCsvFileIntoBeans(String relativePath) { + try (Reader inputReader = new InputStreamReader(new FileInputStream(new File(relativePath)), "UTF-8")) { + BeanListProcessor rowProcessor = new BeanListProcessor(Product.class); + CsvParserSettings settings = new CsvParserSettings(); + settings.setHeaderExtractionEnabled(true); + settings.setProcessor(rowProcessor); + CsvParser parser = new CsvParser(settings); + parser.parse(inputReader); + return rowProcessor.getBeans(); + } catch (IOException e) { + logger.error("IOException opening file: " + relativePath + " " + e.getMessage()); + return new ArrayList(); + } + } + + public List parseCsvFileInBatches(String relativePath) { + try (Reader inputReader = new InputStreamReader(new FileInputStream(new File(relativePath)), "UTF-8")) { + CsvParserSettings settings = new CsvParserSettings(); + settings.setProcessor(new BatchedColumnProcessor(5) { + @Override + public void batchProcessed(int rowsInThisBatch) { + } + }); + CsvParser parser = new CsvParser(settings); + List parsedRows = parser.parseAll(inputReader); + return parsedRows; + } catch (IOException e) { + logger.error("IOException opening file: " + relativePath + " " + e.getMessage()); + return new ArrayList(); + } + } +} diff --git a/libraries-data-2/src/main/java/com/baeldung/univocity/model/Product.java b/libraries-data-2/src/main/java/com/baeldung/univocity/model/Product.java new file mode 100644 index 0000000000..dc4a28f62a --- /dev/null +++ b/libraries-data-2/src/main/java/com/baeldung/univocity/model/Product.java @@ -0,0 +1,44 @@ +package com.baeldung.univocity.model; + +import com.univocity.parsers.annotations.Parsed; + +public class Product { + + @Parsed(field = "product_no") + private String productNumber; + + @Parsed + private String description; + + @Parsed(field = "unit_price") + private float unitPrice; + + public String getProductNumber() { + return productNumber; + } + + public void setProductNumber(String productNumber) { + this.productNumber = productNumber; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public float getUnitPrice() { + return unitPrice; + } + + public void setUnitPrice(float unitPrice) { + this.unitPrice = unitPrice; + } + + @Override + public String toString() { + return "Product [Product Number: " + productNumber + ", Description: " + description + ", Unit Price: " + unitPrice + "]"; + } +} diff --git a/libraries-data-2/src/test/java/com/baeldung/univocity/ParsingServiceUnitTest.java b/libraries-data-2/src/test/java/com/baeldung/univocity/ParsingServiceUnitTest.java new file mode 100644 index 0000000000..8cac0bc4b8 --- /dev/null +++ b/libraries-data-2/src/test/java/com/baeldung/univocity/ParsingServiceUnitTest.java @@ -0,0 +1,112 @@ +package com.baeldung.univocity; + +import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; + +import com.baeldung.univocity.model.Product; + +public class ParsingServiceUnitTest { + + @Test + public void givenCsvFile_thenParsedResultsShouldBeReturned() { + ParsingService parsingService = new ParsingService(); + List productData = parsingService.parseCsvFile("src/test/resources/productList.csv"); + assertEquals(3, productData.size()); + assertEquals(3, productData.get(0).length); + assertEquals("A8993-10", productData.get(0)[0]); + assertEquals("Extra large widget", productData.get(0)[1]); + assertEquals("35.42", productData.get(0)[2]); + assertEquals("D-2938-1", productData.get(1)[0]); + assertEquals("Winding widget \"Deluxe Model\"", productData.get(1)[1]); + assertEquals("245.99", productData.get(1)[2]); + assertEquals("R3212-32", productData.get(2)[0]); + assertEquals("Standard widget", productData.get(2)[1]); + assertEquals("2.34", productData.get(2)[2]); + } + + @Test + public void givenFixedWidthFile_thenParsedResultsShouldBeReturned() { + ParsingService parsingService = new ParsingService(); + List productData = parsingService.parseFixedWidthFile("src/test/resources/productList.txt"); + // Note: any extra spaces on the end will cause a null line to be added + assertEquals(3, productData.size()); + assertEquals(3, productData.get(0).length); + assertEquals("A8993-10", productData.get(0)[0]); + assertEquals("Extra large widget", productData.get(0)[1]); + assertEquals("35.42", productData.get(0)[2]); + assertEquals("D-2938-1", productData.get(1)[0]); + assertEquals("Winding widget \"Deluxe Model\"", productData.get(1)[1]); + assertEquals("245.99", productData.get(1)[2]); + assertEquals("R3212-32", productData.get(2)[0]); + assertEquals("Standard widget", productData.get(2)[1]); + assertEquals("2.34", productData.get(2)[2]); + } + + @Test + public void givenDataAndCsvOutputType_thenCsvFileProduced() { + OutputService outputService = new OutputService(); + List productData = new ArrayList<>(); + productData.add(new Object[] { "1000-3-0", "Widget No. 96", "5.67" }); + productData.add(new Object[] { "G930-M-P", "1/4\" Wocket", ".67" }); + productData.add(new Object[] { "8080-0-M", "No. 54 Jumbo Widget", "35.74" }); + outputService.writeData(productData, OutputService.OutputType.CSV, "src/test/resources/outputProductList.csv"); + + ParsingService parsingService = new ParsingService(); + List writtenData = parsingService.parseCsvFile("src/test/resources/outputProductList.csv"); + assertEquals(3, writtenData.size()); + assertEquals(3, writtenData.get(0).length); + } + + @Test + public void givenDataAndFixedWidthOutputType_thenFixedWidthFileProduced() { + OutputService outputService = new OutputService(); + List productData = new ArrayList<>(); + productData.add(new Object[] { "1000-3-0", "Widget No. 96", "5.67" }); + productData.add(new Object[] { "G930-M-P", "1/4\" Wocket", ".67" }); + productData.add(new Object[] { "8080-0-M", "No. 54 Jumbo Widget", "35.74" }); + outputService.writeData(productData, OutputService.OutputType.FIXED_WIDTH, "src/test/resources/outputProductList.txt"); + + ParsingService parsingService = new ParsingService(); + List writtenData = parsingService.parseFixedWidthFile("src/test/resources/outputProductList.txt"); + assertEquals(3, writtenData.size()); + assertEquals(3, writtenData.get(0).length); + } + + @Test + public void givenCsvFile_thenCsvFileParsedIntoBeans() { + ParsingService parsingService = new ParsingService(); + List products = parsingService.parseCsvFileIntoBeans("src/test/resources/productListWithHeaders.csv"); + assertEquals(2, products.size()); + assertEquals("Product [Product Number: 99-378AG, Description: Wocket Widget #42, Unit Price: 3.56]", products.get(0) + .toString()); + assertEquals("Product [Product Number: TB-333-0, Description: Turbine Widget replacement kit, Unit Price: 800.99]", products.get(1) + .toString()); + } + + @Test + public void givenLisOfProduct_thenWriteFixedWidthFile() { + OutputService outputService = new OutputService(); + Product product = new Product(); + product.setProductNumber("007-PPG0"); + product.setDescription("3/8\" x 1\" Wocket"); + product.setUnitPrice(45.99f); + List products = new ArrayList<>(); + products.add(product); + outputService.writeBeanToFixedWidthFile(products, "src/test/resources/productListWithHeaders.txt"); + ParsingService parsingService = new ParsingService(); + List writtenData = parsingService.parseFixedWidthFile("src/test/resources/productListWithHeaders.txt"); + assertEquals(2, writtenData.size()); + assertEquals(3, writtenData.get(0).length); + } + + @Test + public void givenLargeCsvFile_thenParsedDataShouldBeReturned() { + ParsingService parsingService = new ParsingService(); + List productData = parsingService.parseCsvFileInBatches("src/test/resources/largeProductList.csv"); + assertEquals(36, productData.size()); + } +} diff --git a/libraries-data-2/src/test/resources/largeProductList.csv b/libraries-data-2/src/test/resources/largeProductList.csv new file mode 100644 index 0000000000..2b1813d974 --- /dev/null +++ b/libraries-data-2/src/test/resources/largeProductList.csv @@ -0,0 +1,36 @@ +A8993-10,"Extra large widget",35.42 +D-2938-1,"Winding widget ""Deluxe Model""",245.99 +R3212-32,"Standard widget",2.34 +A8993-10,"Extra large widget",35.42 +D-2938-1,"Winding widget ""Deluxe Model""",245.99 +R3212-32,"Standard widget",2.34 +A8993-10,"Extra large widget",35.42 +D-2938-1,"Winding widget ""Deluxe Model""",245.99 +R3212-32,"Standard widget",2.34 +A8993-10,"Extra large widget",35.42 +D-2938-1,"Winding widget ""Deluxe Model""",245.99 +R3212-32,"Standard widget",2.34 +A8993-10,"Extra large widget",35.42 +D-2938-1,"Winding widget ""Deluxe Model""",245.99 +R3212-32,"Standard widget",2.34 +A8993-10,"Extra large widget",35.42 +D-2938-1,"Winding widget ""Deluxe Model""",245.99 +R3212-32,"Standard widget",2.34 +A8993-10,"Extra large widget",35.42 +D-2938-1,"Winding widget ""Deluxe Model""",245.99 +R3212-32,"Standard widget",2.34 +A8993-10,"Extra large widget",35.42 +D-2938-1,"Winding widget ""Deluxe Model""",245.99 +R3212-32,"Standard widget",2.34 +A8993-10,"Extra large widget",35.42 +D-2938-1,"Winding widget ""Deluxe Model""",245.99 +R3212-32,"Standard widget",2.34 +A8993-10,"Extra large widget",35.42 +D-2938-1,"Winding widget ""Deluxe Model""",245.99 +R3212-32,"Standard widget",2.34 +A8993-10,"Extra large widget",35.42 +D-2938-1,"Winding widget ""Deluxe Model""",245.99 +R3212-32,"Standard widget",2.34 +A8993-10,"Extra large widget",35.42 +D-2938-1,"Winding widget ""Deluxe Model""",245.99 +R3212-32,"Standard widget",2.34 \ No newline at end of file diff --git a/libraries-data-2/src/test/resources/productList.csv b/libraries-data-2/src/test/resources/productList.csv new file mode 100644 index 0000000000..f3a03b6c98 --- /dev/null +++ b/libraries-data-2/src/test/resources/productList.csv @@ -0,0 +1,3 @@ +A8993-10,"Extra large widget",35.42 +D-2938-1,"Winding widget ""Deluxe Model""",245.99 +R3212-32,"Standard widget",2.34 \ No newline at end of file diff --git a/libraries-data-2/src/test/resources/productList.txt b/libraries-data-2/src/test/resources/productList.txt new file mode 100644 index 0000000000..eca4caf55a --- /dev/null +++ b/libraries-data-2/src/test/resources/productList.txt @@ -0,0 +1,3 @@ +A8993-10Extra large widget 35.42 +D-2938-1Winding widget "Deluxe Model" 245.99 +R3212-32Standard widget 2.34 \ No newline at end of file diff --git a/libraries-data-2/src/test/resources/productListWithHeaders.csv b/libraries-data-2/src/test/resources/productListWithHeaders.csv new file mode 100644 index 0000000000..6514a7f0f7 --- /dev/null +++ b/libraries-data-2/src/test/resources/productListWithHeaders.csv @@ -0,0 +1,3 @@ +product_no, description, unit_price +99-378AG,Wocket Widget #42,3.56 +TB-333-0,Turbine Widget replacement kit,800.99 \ No newline at end of file diff --git a/libraries-data-2/src/test/resources/productListWithHeaders.txt b/libraries-data-2/src/test/resources/productListWithHeaders.txt new file mode 100644 index 0000000000..0fced57aed --- /dev/null +++ b/libraries-data-2/src/test/resources/productListWithHeaders.txt @@ -0,0 +1,2 @@ +product_description unit_price +007-PPG03/8" x 1" Wocket 45.99