iSharkFly-Docs/java-tutorials
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #11768 from hkhan/JAVA-9839-cleanup-spark-tests

Browse Source 
[JAVA-9839] Cleanup test files
This commit is contained in:
kwoyke 2022-02-03 15:16:24 +01:00 committed by GitHub
commit 048547c003
1 changed files with 14 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
apache-spark/src/test/java/com/baeldung/differences/rdd/TransformationsUnitTest.java
View File

@ -1,9 +1,9 @@
package com.baeldung.differences.rdd; package com.baeldung.differences.rdd;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDD;
@ -12,12 +12,15 @@ import org.junit.AfterClass;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
import java.io.File;
public class TransformationsUnitTest { public class TransformationsUnitTest {
public static final String COMMA_DELIMITER = ",(?=([^\"]*\"[^\"]*\")*[^\"]*$)"; public static final String COMMA_DELIMITER = ",(?=([^\"]*\"[^\"]*\")*[^\"]*$)";
private static JavaSparkContext sc; private static JavaSparkContext sc;
private static JavaRDD<String> tourists; private static JavaRDD<String> tourists;
@BeforeClass @BeforeClass
public static void init() { public static void init() {
SparkConf conf = new SparkConf().setAppName("uppercaseCountries") SparkConf conf = new SparkConf().setAppName("uppercaseCountries")
@ -25,8 +28,11 @@ public class TransformationsUnitTest {
sc = new JavaSparkContext(conf); sc = new JavaSparkContext(conf);
tourists = sc.textFile("data/Tourist.csv") tourists = sc.textFile("data/Tourist.csv")
.filter(line -> !line.startsWith("Region")); //filter header row .filter(line -> !line.startsWith("Region")); //filter header row
// delete previous output dir and files
FileUtils.deleteQuietly(new File("data/output"));
} }
@AfterClass @AfterClass
public static void cleanup() { public static void cleanup() {
sc.close(); sc.close();
@ -39,9 +45,9 @@ public class TransformationsUnitTest {
return columns[1].toUpperCase(); return columns[1].toUpperCase();
}) })
.distinct(); .distinct();
upperCaseCountries.saveAsTextFile("data/output/uppercase.txt"); upperCaseCountries.saveAsTextFile("data/output/uppercase.txt");
upperCaseCountries.foreach(country -> { upperCaseCountries.foreach(country -> {
//replace non alphanumerical characters //replace non alphanumerical characters
country = country.replaceAll("[^a-zA-Z]", ""); country = country.replaceAll("[^a-zA-Z]", "");
@ -52,9 +58,9 @@ public class TransformationsUnitTest {
@Test @Test
public void whenFilterByCountry_thenShowRequestedCountryRecords() { public void whenFilterByCountry_thenShowRequestedCountryRecords() {
JavaRDD<String> touristsInMexico = tourists.filter(line -> line.split(COMMA_DELIMITER)[1].equals("Mexico")); JavaRDD<String> touristsInMexico = tourists.filter(line -> line.split(COMMA_DELIMITER)[1].equals("Mexico"));
touristsInMexico.saveAsTextFile("data/output/touristInMexico.txt"); touristsInMexico.saveAsTextFile("data/output/touristInMexico.txt");
touristsInMexico.foreach(record -> { touristsInMexico.foreach(record -> {
assertEquals("Mexico", record.split(COMMA_DELIMITER)[1]); assertEquals("Mexico", record.split(COMMA_DELIMITER)[1]);
}); });