JAVA-1848: Moved apache-beam to apache-libraries
This commit is contained in:
sampadawagde
parent
01bbadab2b
commit
31aba66983
|
|
@ -1,3 +0,0 @@
|
||||||
### Relevant Articles:
|
|
||||||
|
|
||||||
- [Introduction to Apache Beam](https://www.baeldung.com/apache-beam)
|
|
||||||
|
|
@ -1,43 +0,0 @@
|
||||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
|
||||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
||||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
||||||
<modelVersion>4.0.0</modelVersion>
|
|
||||||
|
|
||||||
<parent>
|
|
||||||
<groupId>com.baeldung</groupId>
|
|
||||||
<artifactId>parent-modules</artifactId>
|
|
||||||
<version>1.0.0-SNAPSHOT</version>
|
|
||||||
</parent>
|
|
||||||
|
|
||||||
<groupId>com.baeldung.apache</groupId>
|
|
||||||
<artifactId>apache-beam</artifactId>
|
|
||||||
<version>0.0.1-SNAPSHOT</version>
|
|
||||||
|
|
||||||
<dependencies>
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.apache.beam</groupId>
|
|
||||||
<artifactId>beam-sdks-java-core</artifactId>
|
|
||||||
<version>${beam.version}</version>
|
|
||||||
</dependency>
|
|
||||||
<!-- runtime scoped -->
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.apache.beam</groupId>
|
|
||||||
<artifactId>beam-runners-direct-java</artifactId>
|
|
||||||
<version>${beam.version}</version>
|
|
||||||
<scope>runtime</scope>
|
|
||||||
</dependency>
|
|
||||||
<!-- test scoped -->
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.assertj</groupId>
|
|
||||||
<artifactId>assertj-core</artifactId>
|
|
||||||
<version>${assertj.version}</version>
|
|
||||||
<scope>test</scope>
|
|
||||||
</dependency>
|
|
||||||
</dependencies>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<beam.version>2.19.0</beam.version>
|
|
||||||
<assertj.version>3.6.1</assertj.version>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</project>
|
|
||||||
|
|
@ -1,71 +0,0 @@
|
||||||
package com.baeldung.apache.beam.intro;
|
|
||||||
|
|
||||||
import java.util.Arrays;
|
|
||||||
|
|
||||||
import org.apache.beam.sdk.Pipeline;
|
|
||||||
import org.apache.beam.sdk.io.TextIO;
|
|
||||||
import org.apache.beam.sdk.options.PipelineOptions;
|
|
||||||
import org.apache.beam.sdk.options.PipelineOptionsFactory;
|
|
||||||
import org.apache.beam.sdk.transforms.Count;
|
|
||||||
import org.apache.beam.sdk.transforms.Filter;
|
|
||||||
import org.apache.beam.sdk.transforms.FlatMapElements;
|
|
||||||
import org.apache.beam.sdk.transforms.MapElements;
|
|
||||||
import org.apache.beam.sdk.values.KV;
|
|
||||||
import org.apache.beam.sdk.values.PCollection;
|
|
||||||
import org.apache.beam.sdk.values.TypeDescriptors;
|
|
||||||
|
|
||||||
public class WordCount {
|
|
||||||
|
|
||||||
public static boolean wordCount(String inputFilePath, String outputFilePath) {
|
|
||||||
// We use default options
|
|
||||||
PipelineOptions options = PipelineOptionsFactory.create();
|
|
||||||
// to create the pipeline
|
|
||||||
Pipeline p = Pipeline.create(options);
|
|
||||||
// Here is our workflow graph
|
|
||||||
PCollection<KV<String, Long>> wordCount = p
|
|
||||||
.apply("(1) Read all lines", TextIO.read().from(inputFilePath))
|
|
||||||
.apply("(2) Flatmap to a list of words", FlatMapElements.into(TypeDescriptors.strings())
|
|
||||||
.via(line -> Arrays.asList(line.split("\\s"))))
|
|
||||||
.apply("(3) Lowercase all", MapElements.into(TypeDescriptors.strings())
|
|
||||||
.via(word -> word.toLowerCase()))
|
|
||||||
.apply("(4) Trim punctuations", MapElements.into(TypeDescriptors.strings())
|
|
||||||
.via(word -> trim(word)))
|
|
||||||
.apply("(5) Filter stopwords", Filter.by(word -> !isStopWord(word)))
|
|
||||||
.apply("(6) Count words", Count.perElement());
|
|
||||||
// We convert the PCollection to String so that we can write it to file
|
|
||||||
wordCount.apply(MapElements.into(TypeDescriptors.strings())
|
|
||||||
.via(count -> count.getKey() + " --> " + count.getValue()))
|
|
||||||
.apply(TextIO.write().to(outputFilePath));
|
|
||||||
// Finally we must run the pipeline, otherwise it's only a definition
|
|
||||||
p.run().waitUntilFinish();
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static boolean isStopWord(String word) {
|
|
||||||
String[] stopwords = {"am", "are", "is", "i", "you", "me",
|
|
||||||
"he", "she", "they", "them", "was",
|
|
||||||
"were", "from", "in", "of", "to", "be",
|
|
||||||
"him", "her", "us", "and", "or"};
|
|
||||||
for (String stopword : stopwords) {
|
|
||||||
if (stopword.compareTo(word) == 0) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static String trim(String word) {
|
|
||||||
return word.replace("(","")
|
|
||||||
.replace(")", "")
|
|
||||||
.replace(",", "")
|
|
||||||
.replace(".", "")
|
|
||||||
.replace("\"", "")
|
|
||||||
.replace("'", "")
|
|
||||||
.replace(":", "")
|
|
||||||
.replace(";", "")
|
|
||||||
.replace("-", "")
|
|
||||||
.replace("?", "")
|
|
||||||
.replace("!", "");
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
@ -1,19 +0,0 @@
|
||||||
package com.baeldung.apache.beam.intro;
|
|
||||||
|
|
||||||
import static org.junit.Assert.assertTrue;
|
|
||||||
|
|
||||||
import org.junit.Ignore;
|
|
||||||
import org.junit.Test;
|
|
||||||
|
|
||||||
import com.baeldung.apache.beam.intro.WordCount;
|
|
||||||
|
|
||||||
public class WordCountUnitTest {
|
|
||||||
|
|
||||||
@Test
|
|
||||||
// @Ignore
|
|
||||||
public void givenInputFile_whenWordCountRuns_thenJobFinishWithoutError() {
|
|
||||||
boolean jobDone = WordCount.wordCount("src/test/resources/wordcount.txt", "target/output");
|
|
||||||
assertTrue(jobDone);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
@ -1,16 +0,0 @@
|
||||||
We've all heard the scare stories about North Korea: the homemade nuclear arsenal built while their people starve and then aimed imprecisely at the rest of the world, a
|
|
||||||
leader so deluded he makes L Ron Hubbard look like a man excessively overburdened with self-doubt and their deep-seated belief that foreign capitalists will invade at any
|
|
||||||
moment and steal all their bauxite.
|
|
||||||
The popular portrayal of this Marxist nation is something like one of the more harrowing episodes of M*A*S*H, only with the cast of wacky characters replaced by twitchy,
|
|
||||||
heavily armed Stalinist meth addicts
|
|
||||||
Cracked would like to take a moment to celebrate the good things about North Korea though, the things that the country's enemies prefer to suppress as part of their politically
|
|
||||||
motivated jealousy. Like how no different to you and me, there's nothing every North Korean likes more after an 18 hour shift at the phosphorus plant than a nice beer to go with
|
|
||||||
his dried fish ration. Ever attentive to its people's needs and in the twinkling of a decade, North Korea's leadership bought, disassembled, transported and rebuilt a British
|
|
||||||
brewery in order to discover and reproduce the secrets of beer and then brew the sweet nectar for its hardworking people, up to 18 bottles at a time. And with minimal fatalities.
|
|
||||||
When was the last time YOUR leader got a beer for YOU, American? (NB do not answer this question if you are Henry Louis Gates).
|
|
||||||
Or how about the fried chicken restaurant that downtown Pyongyang boasts? Yes real chicken, fried and then delivered to your sleeping cube, with optional beer if you like! You
|
|
||||||
don't even have to remove the feathers or pull out the gizzard yourself. Mostly. Americans must eat their fried chicken from a bucket, like swine, sold by a company so secretive
|
|
||||||
that even the very blend of seasoning used is intentionally kept from them. And they call North Korea paranoid?
|
|
||||||
And how many nations would entertain the syphilitic, bourgeois ramblings of Bill Clinton let alone permit him anywhere near their proud womenfolk? Only wise Kim Jong Il could see
|
|
||||||
past Bill's many, many imperfections and treat him with the pity and kindness he deserves, accepting his feeble pleas to pardon the American spies rightly convicted of photographing
|
|
||||||
the nation's sensitive beetroot fields.
|
|
||||||
Loading…
Reference in New Issue