[JAVA-11122] Logging clean up
This commit is contained in:
parent
fe96f9747f
commit
39e01903c2
|
@ -12,13 +12,18 @@ import org.junit.AfterClass;
|
||||||
import org.junit.BeforeClass;
|
import org.junit.BeforeClass;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
import scala.Tuple2;
|
import scala.Tuple2;
|
||||||
|
|
||||||
public class ActionsUnitTest {
|
public class ActionsUnitTest {
|
||||||
|
|
||||||
|
public static final Logger LOG = LoggerFactory.getLogger(ActionsUnitTest.class);
|
||||||
|
|
||||||
private static JavaRDD<String> tourists;
|
private static JavaRDD<String> tourists;
|
||||||
private static JavaSparkContext sc;
|
private static JavaSparkContext sc;
|
||||||
public static final String COMMA_DELIMITER = ",(?=([^\"]*\"[^\"]*\")*[^\"]*$)";
|
public static final String COMMA_DELIMITER = ",(?=([^\"]*\"[^\"]*\")*[^\"]*$)";
|
||||||
|
|
||||||
@BeforeClass
|
@BeforeClass
|
||||||
public static void init() {
|
public static void init() {
|
||||||
SparkConf conf = new SparkConf().setAppName("reduce")
|
SparkConf conf = new SparkConf().setAppName("reduce")
|
||||||
|
@ -26,7 +31,7 @@ public class ActionsUnitTest {
|
||||||
sc = new JavaSparkContext(conf);
|
sc = new JavaSparkContext(conf);
|
||||||
tourists = sc.textFile("data/Tourist.csv").filter(line -> !line.startsWith("Region"));
|
tourists = sc.textFile("data/Tourist.csv").filter(line -> !line.startsWith("Region"));
|
||||||
}
|
}
|
||||||
|
|
||||||
@AfterClass
|
@AfterClass
|
||||||
public static void cleanup() {
|
public static void cleanup() {
|
||||||
sc.close();
|
sc.close();
|
||||||
|
@ -40,11 +45,11 @@ public class ActionsUnitTest {
|
||||||
})
|
})
|
||||||
.distinct();
|
.distinct();
|
||||||
Long numberOfCountries = countries.count();
|
Long numberOfCountries = countries.count();
|
||||||
System.out.println("Count: " + numberOfCountries);
|
LOG.debug("Count: {}", numberOfCountries);
|
||||||
|
|
||||||
assertEquals(Long.valueOf(220), numberOfCountries);
|
assertEquals(Long.valueOf(220), numberOfCountries);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void whenReduceByKeySum_thenTotalValuePerKey() {
|
public void whenReduceByKeySum_thenTotalValuePerKey() {
|
||||||
JavaRDD<String> touristsExpenditure = tourists.filter(line -> line.split(COMMA_DELIMITER)[3].contains("expenditure"));
|
JavaRDD<String> touristsExpenditure = tourists.filter(line -> line.split(COMMA_DELIMITER)[3].contains("expenditure"));
|
||||||
|
@ -53,10 +58,12 @@ public class ActionsUnitTest {
|
||||||
String[] columns = line.split(COMMA_DELIMITER);
|
String[] columns = line.split(COMMA_DELIMITER);
|
||||||
return new Tuple2<>(columns[1], Double.valueOf(columns[6]));
|
return new Tuple2<>(columns[1], Double.valueOf(columns[6]));
|
||||||
});
|
});
|
||||||
List<Tuple2<String, Double>> totalByCountry = expenditurePairRdd.reduceByKey((x, y) -> x + y)
|
List<Tuple2<String, Double>> totalByCountry = expenditurePairRdd
|
||||||
.collect();
|
.reduceByKey(Double::sum)
|
||||||
System.out.println("Total per Country: " + totalByCountry);
|
.collect();
|
||||||
|
|
||||||
|
LOG.debug("Total per Country: {}", totalByCountry);
|
||||||
|
|
||||||
for(Tuple2<String, Double> tuple : totalByCountry) {
|
for(Tuple2<String, Double> tuple : totalByCountry) {
|
||||||
if (tuple._1.equals("Mexico")) {
|
if (tuple._1.equals("Mexico")) {
|
||||||
assertEquals(Double.valueOf(99164), tuple._2);
|
assertEquals(Double.valueOf(99164), tuple._2);
|
||||||
|
|
|
@ -39,8 +39,10 @@ public class DataFrameUnitTest {
|
||||||
@Test
|
@Test
|
||||||
public void whenSelectSpecificColumns_thenColumnsFiltered() {
|
public void whenSelectSpecificColumns_thenColumnsFiltered() {
|
||||||
Dataset<Row> selectedData = data.select(col("country"), col("year"), col("value"));
|
Dataset<Row> selectedData = data.select(col("country"), col("year"), col("value"));
|
||||||
selectedData.show();
|
|
||||||
|
// uncomment to see table
|
||||||
|
// selectedData.show();
|
||||||
|
|
||||||
List<String> resultList = Arrays.asList(selectedData.columns());
|
List<String> resultList = Arrays.asList(selectedData.columns());
|
||||||
assertTrue(resultList.contains("country"));
|
assertTrue(resultList.contains("country"));
|
||||||
assertTrue(resultList.contains("year"));
|
assertTrue(resultList.contains("year"));
|
||||||
|
@ -52,22 +54,26 @@ public class DataFrameUnitTest {
|
||||||
@Test
|
@Test
|
||||||
public void whenFilteringByCountry_thenCountryRecordsSelected() {
|
public void whenFilteringByCountry_thenCountryRecordsSelected() {
|
||||||
Dataset<Row> filteredData = data.filter(col("country").equalTo("Mexico"));
|
Dataset<Row> filteredData = data.filter(col("country").equalTo("Mexico"));
|
||||||
filteredData.show();
|
|
||||||
|
// uncomment to see table
|
||||||
|
// filteredData.show();
|
||||||
|
|
||||||
filteredData.foreach(record -> {
|
filteredData.foreach(record -> {
|
||||||
assertEquals("Mexico", record.get(1));
|
assertEquals("Mexico", record.get(1));
|
||||||
});
|
});
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void whenGroupCountByCountry_thenContryTotalRecords() {
|
public void whenGroupCountByCountry_thenContryTotalRecords() {
|
||||||
Dataset<Row> recordsPerCountry = data.groupBy(col("country"))
|
Dataset<Row> recordsPerCountry = data.groupBy(col("country"))
|
||||||
.count();
|
.count();
|
||||||
recordsPerCountry.show();
|
|
||||||
|
// uncomment to see table
|
||||||
|
// recordsPerCountry.show();
|
||||||
|
|
||||||
Dataset<Row> filteredData = recordsPerCountry.filter(col("country").equalTo("Sweden"));
|
Dataset<Row> filteredData = recordsPerCountry.filter(col("country").equalTo("Sweden"));
|
||||||
assertEquals(new Long(12), filteredData.first()
|
assertEquals(12L, filteredData.first()
|
||||||
.get(1));
|
.get(1));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -3,6 +3,7 @@ package com.baeldung.differences.rdd;
|
||||||
import static org.apache.spark.sql.functions.col;
|
import static org.apache.spark.sql.functions.col;
|
||||||
import static org.apache.spark.sql.functions.sum;
|
import static org.apache.spark.sql.functions.sum;
|
||||||
import static org.junit.Assert.assertEquals;
|
import static org.junit.Assert.assertEquals;
|
||||||
|
import static org.junit.Assert.assertTrue;
|
||||||
|
|
||||||
import org.apache.spark.api.java.function.FilterFunction;
|
import org.apache.spark.api.java.function.FilterFunction;
|
||||||
import org.apache.spark.sql.DataFrameReader;
|
import org.apache.spark.sql.DataFrameReader;
|
||||||
|
@ -29,8 +30,8 @@ public class DatasetUnitTest {
|
||||||
DataFrameReader dataFrameReader = session.read();
|
DataFrameReader dataFrameReader = session.read();
|
||||||
Dataset<Row> data = dataFrameReader.option("header", "true")
|
Dataset<Row> data = dataFrameReader.option("header", "true")
|
||||||
.csv("data/Tourist.csv");
|
.csv("data/Tourist.csv");
|
||||||
Dataset<Row> responseWithSelectedColumns = data.select(col("region"),
|
Dataset<Row> responseWithSelectedColumns = data.select(col("region"),
|
||||||
col("country"), col("year"), col("series"), col("value").cast("double"),
|
col("country"), col("year"), col("series"), col("value").cast("double"),
|
||||||
col("footnotes"), col("source"));
|
col("footnotes"), col("source"));
|
||||||
typedDataset = responseWithSelectedColumns.as(Encoders.bean(TouristData.class));
|
typedDataset = responseWithSelectedColumns.as(Encoders.bean(TouristData.class));
|
||||||
}
|
}
|
||||||
|
@ -45,7 +46,9 @@ public class DatasetUnitTest {
|
||||||
Dataset<TouristData> selectedData = typedDataset
|
Dataset<TouristData> selectedData = typedDataset
|
||||||
.filter((FilterFunction<TouristData>) record -> record.getCountry()
|
.filter((FilterFunction<TouristData>) record -> record.getCountry()
|
||||||
.equals("Norway"));
|
.equals("Norway"));
|
||||||
selectedData.show();
|
|
||||||
|
// uncomment to see output
|
||||||
|
// selectedData.show();
|
||||||
|
|
||||||
selectedData.foreach(record -> {
|
selectedData.foreach(record -> {
|
||||||
assertEquals("Norway", record.getCountry());
|
assertEquals("Norway", record.getCountry());
|
||||||
|
@ -56,28 +59,41 @@ public class DatasetUnitTest {
|
||||||
public void whenGroupCountByCountry_thenContryTotalRecords() {
|
public void whenGroupCountByCountry_thenContryTotalRecords() {
|
||||||
Dataset<Row> countriesCount = typedDataset.groupBy(typedDataset.col("country"))
|
Dataset<Row> countriesCount = typedDataset.groupBy(typedDataset.col("country"))
|
||||||
.count();
|
.count();
|
||||||
countriesCount.show();
|
|
||||||
|
|
||||||
assertEquals(Long.valueOf(220), Long.valueOf(countriesCount.count()));
|
// uncomment to see output
|
||||||
|
// countriesCount.show();
|
||||||
|
|
||||||
|
assertEquals(220, countriesCount.count());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void whenFilteredByPropertyRange_thenRetreiveValidRecords() {
|
public void whenFilteredByPropertyRange_thenRetreiveValidRecords() {
|
||||||
// Filter records with existing data for years between 2010 and 2017
|
// Filter records with existing data for years between 2010 and 2017
|
||||||
typedDataset.filter((FilterFunction<TouristData>) record -> record.getYear() != null
|
Dataset<TouristData> filteredData = typedDataset.filter(
|
||||||
&& (Long.valueOf(record.getYear()) > 2010 && Long.valueOf(record.getYear()) < 2017))
|
(FilterFunction<TouristData>) record -> record.getYear() != null
|
||||||
.show();
|
&& (Long.parseLong(record.getYear()) > 2010 && Long.parseLong(record.getYear()) < 2017));
|
||||||
|
|
||||||
|
// uncomment to see output
|
||||||
|
// filteredData.show();
|
||||||
|
|
||||||
|
assertEquals(394, filteredData.count());
|
||||||
|
filteredData.foreach(record -> {
|
||||||
|
assertTrue(Integer.parseInt(record.getYear()) > 2010 && Integer.parseInt(record.getYear()) < 2017);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void whenSumValue_thenRetreiveTotalValue() {
|
public void whenSumValue_thenRetreiveTotalValue() {
|
||||||
// Total tourist expenditure by country
|
// Total tourist expenditure by country
|
||||||
typedDataset.filter((FilterFunction<TouristData>) record -> record.getValue() != null
|
Dataset<Row> filteredData = typedDataset.filter((FilterFunction<TouristData>) record -> record.getValue() != null
|
||||||
&& record.getSeries()
|
&& record.getSeries().contains("expenditure"))
|
||||||
.contains("expenditure"))
|
.groupBy("country")
|
||||||
.groupBy("country")
|
.agg(sum("value"));
|
||||||
.agg(sum("value"))
|
|
||||||
.show();
|
// uncomment to see output
|
||||||
|
// filteredData.show();
|
||||||
|
|
||||||
|
assertEquals(212, filteredData.count());
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,12 +3,13 @@ package com.baeldung.graphql;
|
||||||
import org.junit.jupiter.api.AfterAll;
|
import org.junit.jupiter.api.AfterAll;
|
||||||
import org.junit.jupiter.api.BeforeAll;
|
import org.junit.jupiter.api.BeforeAll;
|
||||||
import org.mockserver.client.MockServerClient;
|
import org.mockserver.client.MockServerClient;
|
||||||
|
import org.mockserver.configuration.Configuration;
|
||||||
import org.mockserver.integration.ClientAndServer;
|
import org.mockserver.integration.ClientAndServer;
|
||||||
import org.mockserver.model.HttpStatusCode;
|
import org.mockserver.model.HttpStatusCode;
|
||||||
|
import org.slf4j.event.Level;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.ServerSocket;
|
import java.net.ServerSocket;
|
||||||
import java.net.URISyntaxException;
|
|
||||||
|
|
||||||
import static org.mockserver.integration.ClientAndServer.startClientAndServer;
|
import static org.mockserver.integration.ClientAndServer.startClientAndServer;
|
||||||
import static org.mockserver.matchers.Times.exactly;
|
import static org.mockserver.matchers.Times.exactly;
|
||||||
|
@ -17,20 +18,22 @@ import static org.mockserver.model.HttpResponse.response;
|
||||||
|
|
||||||
public class GraphQLMockServer {
|
public class GraphQLMockServer {
|
||||||
|
|
||||||
public static ClientAndServer mockServer;
|
private static final String SERVER_ADDRESS = "127.0.0.1";
|
||||||
|
private static final String PATH = "/graphql";
|
||||||
|
|
||||||
public static String serviceUrl;
|
public static String serviceUrl;
|
||||||
|
|
||||||
|
private static ClientAndServer mockServer;
|
||||||
private static int serverPort;
|
private static int serverPort;
|
||||||
|
|
||||||
public static final String SERVER_ADDRESS = "127.0.0.1";
|
|
||||||
public static final String HTTP_GET_POST = "GET";
|
|
||||||
public static final String PATH = "/graphql";
|
|
||||||
|
|
||||||
@BeforeAll
|
@BeforeAll
|
||||||
static void startServer() throws IOException, URISyntaxException {
|
static void startServer() throws IOException {
|
||||||
serverPort = getFreePort();
|
serverPort = getFreePort();
|
||||||
serviceUrl = "http://" + SERVER_ADDRESS + ":" + serverPort + PATH;
|
serviceUrl = "http://" + SERVER_ADDRESS + ":" + serverPort + PATH;
|
||||||
mockServer = startClientAndServer(serverPort);
|
|
||||||
|
Configuration config = Configuration.configuration().logLevel(Level.WARN);
|
||||||
|
mockServer = startClientAndServer(config, serverPort);
|
||||||
|
|
||||||
mockAllBooksTitleRequest();
|
mockAllBooksTitleRequest();
|
||||||
mockAllBooksTitleAuthorRequest();
|
mockAllBooksTitleAuthorRequest();
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue