[JAVA-11122] Logging clean up

This commit is contained in:
Haroon Khan 2022-05-11 12:28:14 +01:00
parent fe96f9747f
commit 39e01903c2
4 changed files with 72 additions and 40 deletions

View File

@ -12,13 +12,18 @@ import org.junit.AfterClass;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Tuple2; import scala.Tuple2;
public class ActionsUnitTest { public class ActionsUnitTest {
public static final Logger LOG = LoggerFactory.getLogger(ActionsUnitTest.class);
private static JavaRDD<String> tourists; private static JavaRDD<String> tourists;
private static JavaSparkContext sc; private static JavaSparkContext sc;
public static final String COMMA_DELIMITER = ",(?=([^\"]*\"[^\"]*\")*[^\"]*$)"; public static final String COMMA_DELIMITER = ",(?=([^\"]*\"[^\"]*\")*[^\"]*$)";
@BeforeClass @BeforeClass
public static void init() { public static void init() {
SparkConf conf = new SparkConf().setAppName("reduce") SparkConf conf = new SparkConf().setAppName("reduce")
@ -26,7 +31,7 @@ public class ActionsUnitTest {
sc = new JavaSparkContext(conf); sc = new JavaSparkContext(conf);
tourists = sc.textFile("data/Tourist.csv").filter(line -> !line.startsWith("Region")); tourists = sc.textFile("data/Tourist.csv").filter(line -> !line.startsWith("Region"));
} }
@AfterClass @AfterClass
public static void cleanup() { public static void cleanup() {
sc.close(); sc.close();
@ -40,11 +45,11 @@ public class ActionsUnitTest {
}) })
.distinct(); .distinct();
Long numberOfCountries = countries.count(); Long numberOfCountries = countries.count();
System.out.println("Count: " + numberOfCountries); LOG.debug("Count: {}", numberOfCountries);
assertEquals(Long.valueOf(220), numberOfCountries); assertEquals(Long.valueOf(220), numberOfCountries);
} }
@Test @Test
public void whenReduceByKeySum_thenTotalValuePerKey() { public void whenReduceByKeySum_thenTotalValuePerKey() {
JavaRDD<String> touristsExpenditure = tourists.filter(line -> line.split(COMMA_DELIMITER)[3].contains("expenditure")); JavaRDD<String> touristsExpenditure = tourists.filter(line -> line.split(COMMA_DELIMITER)[3].contains("expenditure"));
@ -53,10 +58,12 @@ public class ActionsUnitTest {
String[] columns = line.split(COMMA_DELIMITER); String[] columns = line.split(COMMA_DELIMITER);
return new Tuple2<>(columns[1], Double.valueOf(columns[6])); return new Tuple2<>(columns[1], Double.valueOf(columns[6]));
}); });
List<Tuple2<String, Double>> totalByCountry = expenditurePairRdd.reduceByKey((x, y) -> x + y) List<Tuple2<String, Double>> totalByCountry = expenditurePairRdd
.collect(); .reduceByKey(Double::sum)
System.out.println("Total per Country: " + totalByCountry); .collect();
LOG.debug("Total per Country: {}", totalByCountry);
for(Tuple2<String, Double> tuple : totalByCountry) { for(Tuple2<String, Double> tuple : totalByCountry) {
if (tuple._1.equals("Mexico")) { if (tuple._1.equals("Mexico")) {
assertEquals(Double.valueOf(99164), tuple._2); assertEquals(Double.valueOf(99164), tuple._2);

View File

@ -39,8 +39,10 @@ public class DataFrameUnitTest {
@Test @Test
public void whenSelectSpecificColumns_thenColumnsFiltered() { public void whenSelectSpecificColumns_thenColumnsFiltered() {
Dataset<Row> selectedData = data.select(col("country"), col("year"), col("value")); Dataset<Row> selectedData = data.select(col("country"), col("year"), col("value"));
selectedData.show();
// uncomment to see table
// selectedData.show();
List<String> resultList = Arrays.asList(selectedData.columns()); List<String> resultList = Arrays.asList(selectedData.columns());
assertTrue(resultList.contains("country")); assertTrue(resultList.contains("country"));
assertTrue(resultList.contains("year")); assertTrue(resultList.contains("year"));
@ -52,22 +54,26 @@ public class DataFrameUnitTest {
@Test @Test
public void whenFilteringByCountry_thenCountryRecordsSelected() { public void whenFilteringByCountry_thenCountryRecordsSelected() {
Dataset<Row> filteredData = data.filter(col("country").equalTo("Mexico")); Dataset<Row> filteredData = data.filter(col("country").equalTo("Mexico"));
filteredData.show();
// uncomment to see table
// filteredData.show();
filteredData.foreach(record -> { filteredData.foreach(record -> {
assertEquals("Mexico", record.get(1)); assertEquals("Mexico", record.get(1));
}); });
} }
@Test @Test
public void whenGroupCountByCountry_thenContryTotalRecords() { public void whenGroupCountByCountry_thenContryTotalRecords() {
Dataset<Row> recordsPerCountry = data.groupBy(col("country")) Dataset<Row> recordsPerCountry = data.groupBy(col("country"))
.count(); .count();
recordsPerCountry.show();
// uncomment to see table
// recordsPerCountry.show();
Dataset<Row> filteredData = recordsPerCountry.filter(col("country").equalTo("Sweden")); Dataset<Row> filteredData = recordsPerCountry.filter(col("country").equalTo("Sweden"));
assertEquals(new Long(12), filteredData.first() assertEquals(12L, filteredData.first()
.get(1)); .get(1));
} }

View File

@ -3,6 +3,7 @@ package com.baeldung.differences.rdd;
import static org.apache.spark.sql.functions.col; import static org.apache.spark.sql.functions.col;
import static org.apache.spark.sql.functions.sum; import static org.apache.spark.sql.functions.sum;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.apache.spark.api.java.function.FilterFunction; import org.apache.spark.api.java.function.FilterFunction;
import org.apache.spark.sql.DataFrameReader; import org.apache.spark.sql.DataFrameReader;
@ -29,8 +30,8 @@ public class DatasetUnitTest {
DataFrameReader dataFrameReader = session.read(); DataFrameReader dataFrameReader = session.read();
Dataset<Row> data = dataFrameReader.option("header", "true") Dataset<Row> data = dataFrameReader.option("header", "true")
.csv("data/Tourist.csv"); .csv("data/Tourist.csv");
Dataset<Row> responseWithSelectedColumns = data.select(col("region"), Dataset<Row> responseWithSelectedColumns = data.select(col("region"),
col("country"), col("year"), col("series"), col("value").cast("double"), col("country"), col("year"), col("series"), col("value").cast("double"),
col("footnotes"), col("source")); col("footnotes"), col("source"));
typedDataset = responseWithSelectedColumns.as(Encoders.bean(TouristData.class)); typedDataset = responseWithSelectedColumns.as(Encoders.bean(TouristData.class));
} }
@ -45,7 +46,9 @@ public class DatasetUnitTest {
Dataset<TouristData> selectedData = typedDataset Dataset<TouristData> selectedData = typedDataset
.filter((FilterFunction<TouristData>) record -> record.getCountry() .filter((FilterFunction<TouristData>) record -> record.getCountry()
.equals("Norway")); .equals("Norway"));
selectedData.show();
// uncomment to see output
// selectedData.show();
selectedData.foreach(record -> { selectedData.foreach(record -> {
assertEquals("Norway", record.getCountry()); assertEquals("Norway", record.getCountry());
@ -56,28 +59,41 @@ public class DatasetUnitTest {
public void whenGroupCountByCountry_thenContryTotalRecords() { public void whenGroupCountByCountry_thenContryTotalRecords() {
Dataset<Row> countriesCount = typedDataset.groupBy(typedDataset.col("country")) Dataset<Row> countriesCount = typedDataset.groupBy(typedDataset.col("country"))
.count(); .count();
countriesCount.show();
assertEquals(Long.valueOf(220), Long.valueOf(countriesCount.count())); // uncomment to see output
// countriesCount.show();
assertEquals(220, countriesCount.count());
} }
@Test @Test
public void whenFilteredByPropertyRange_thenRetreiveValidRecords() { public void whenFilteredByPropertyRange_thenRetreiveValidRecords() {
// Filter records with existing data for years between 2010 and 2017 // Filter records with existing data for years between 2010 and 2017
typedDataset.filter((FilterFunction<TouristData>) record -> record.getYear() != null Dataset<TouristData> filteredData = typedDataset.filter(
&& (Long.valueOf(record.getYear()) > 2010 && Long.valueOf(record.getYear()) < 2017)) (FilterFunction<TouristData>) record -> record.getYear() != null
.show(); && (Long.parseLong(record.getYear()) > 2010 && Long.parseLong(record.getYear()) < 2017));
// uncomment to see output
// filteredData.show();
assertEquals(394, filteredData.count());
filteredData.foreach(record -> {
assertTrue(Integer.parseInt(record.getYear()) > 2010 && Integer.parseInt(record.getYear()) < 2017);
});
} }
@Test @Test
public void whenSumValue_thenRetreiveTotalValue() { public void whenSumValue_thenRetreiveTotalValue() {
// Total tourist expenditure by country // Total tourist expenditure by country
typedDataset.filter((FilterFunction<TouristData>) record -> record.getValue() != null Dataset<Row> filteredData = typedDataset.filter((FilterFunction<TouristData>) record -> record.getValue() != null
&& record.getSeries() && record.getSeries().contains("expenditure"))
.contains("expenditure")) .groupBy("country")
.groupBy("country") .agg(sum("value"));
.agg(sum("value"))
.show(); // uncomment to see output
// filteredData.show();
assertEquals(212, filteredData.count());
} }
} }

View File

@ -3,12 +3,13 @@ package com.baeldung.graphql;
import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeAll;
import org.mockserver.client.MockServerClient; import org.mockserver.client.MockServerClient;
import org.mockserver.configuration.Configuration;
import org.mockserver.integration.ClientAndServer; import org.mockserver.integration.ClientAndServer;
import org.mockserver.model.HttpStatusCode; import org.mockserver.model.HttpStatusCode;
import org.slf4j.event.Level;
import java.io.IOException; import java.io.IOException;
import java.net.ServerSocket; import java.net.ServerSocket;
import java.net.URISyntaxException;
import static org.mockserver.integration.ClientAndServer.startClientAndServer; import static org.mockserver.integration.ClientAndServer.startClientAndServer;
import static org.mockserver.matchers.Times.exactly; import static org.mockserver.matchers.Times.exactly;
@ -17,20 +18,22 @@ import static org.mockserver.model.HttpResponse.response;
public class GraphQLMockServer { public class GraphQLMockServer {
public static ClientAndServer mockServer; private static final String SERVER_ADDRESS = "127.0.0.1";
private static final String PATH = "/graphql";
public static String serviceUrl; public static String serviceUrl;
private static ClientAndServer mockServer;
private static int serverPort; private static int serverPort;
public static final String SERVER_ADDRESS = "127.0.0.1";
public static final String HTTP_GET_POST = "GET";
public static final String PATH = "/graphql";
@BeforeAll @BeforeAll
static void startServer() throws IOException, URISyntaxException { static void startServer() throws IOException {
serverPort = getFreePort(); serverPort = getFreePort();
serviceUrl = "http://" + SERVER_ADDRESS + ":" + serverPort + PATH; serviceUrl = "http://" + SERVER_ADDRESS + ":" + serverPort + PATH;
mockServer = startClientAndServer(serverPort);
Configuration config = Configuration.configuration().logLevel(Level.WARN);
mockServer = startClientAndServer(config, serverPort);
mockAllBooksTitleRequest(); mockAllBooksTitleRequest();
mockAllBooksTitleAuthorRequest(); mockAllBooksTitleAuthorRequest();
} }