iSharkFly-Docs
/
java-tutorials
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

BAEL-1175 - Finalized example and improved scripts

This commit is contained in:
Thoughtscript 2018-01-15 05:01:38 +00:00
parent 36bab29858
commit 11bcac7159
11 changed files with 161 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
spring-cloud/spring-cloud-stream-starters/bash/hadoop.sh
View File

@ -1,3 +1,5 @@
#!/usr/bin/env bash
# For Ubuntu 14.04
# Inspired from: https://github.com/curran/setupHadoop/blob/master/setupHadoop.sh
# Use from the user directory
@ -32,7 +34,7 @@ sudo mv ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys
# SSH
ssh localhost
# Authenticate with local user
# authenticate with osboxes.org
# Start NameNode daemon and DataNode daemon
start-dfs.sh
@ -40,3 +42,5 @@ start-dfs.sh
# Install Maven
sudo apt-get install maven
# Access Hadoop - http://localhost:50070

3
spring-cloud/spring-cloud-stream-starters/boot/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
.idea
*/target/*
*.iml

55
spring-cloud/spring-cloud-stream-starters/boot/pom.xml Normal file
View File

@ -0,0 +1,55 @@
<?xml version="1.0"?>
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0">
<modelVersion>4.0.0</modelVersion>
<groupId>com.baeldung.twitterhdfs</groupId>
<artifactId>twitterhdfs</artifactId>
<packaging>jar</packaging>
<version>1.0.0</version>
<name>twitterhdfs</name>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>1.8</java.version>
</properties>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>1.5.8.RELEASE</version>
</parent>
<dependencies>
<!-- Spring Stream Starter Apps -->
<dependency>
<groupId>org.springframework.cloud.stream.app</groupId>
<artifactId>spring-cloud-starter-stream-source-twitterstream</artifactId>
<version>1.3.1.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework.cloud.stream.app</groupId>
<artifactId>spring-cloud-starter-stream-sink-hdfs</artifactId>
<version>1.3.1.RELEASE</version>
</dependency>
<!-- JSTL/JSP -->
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>jstl</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
<finalName>twitterhdfs</finalName>
</build>
</project>

18
spring-cloud/spring-cloud-stream-starters/boot/src/main/java/com/baeldung/twitterhdfs/aggregate/AggregateApp.java Normal file
View File

@ -0,0 +1,18 @@
package com.baeldung.twitterhdfs.aggregate;
import com.baeldung.twitterhdfs.processor.ProcessorApp;
import com.baeldung.twitterhdfs.source.SourceApp;
import com.baeldung.twitterhdfs.sink.SinkApp;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.stream.aggregate.AggregateApplicationBuilder;
@SpringBootApplication
public class AggregateApp {
public static void main(String[] args) {
new AggregateApplicationBuilder()
.from(SourceApp.class).args("--fixedDelay=5000")
.via(ProcessorApp.class)
.to(SinkApp.class).args("--debug=true")
.run(args);
}
}

20
spring-cloud/spring-cloud-stream-starters/boot/src/main/java/com/baeldung/twitterhdfs/processor/ProcessorApp.java Normal file
View File

@ -0,0 +1,20 @@
package com.baeldung.twitterhdfs.processor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.stream.annotation.EnableBinding;
import org.springframework.cloud.stream.messaging.Processor;
import org.springframework.integration.annotation.Transformer;
@SpringBootApplication
@EnableBinding(Processor.class)
public class ProcessorApp {
Logger log = LoggerFactory.getLogger(ProcessorApp.class);
@Transformer(inputChannel = Processor.INPUT, outputChannel = Processor.OUTPUT)
public String processMessage(String payload) {
log.info("Payload received!");
return payload;
}
}

22
spring-cloud/spring-cloud-stream-starters/boot/src/main/java/com/baeldung/twitterhdfs/sink/SinkApp.java Normal file
View File

@ -0,0 +1,22 @@
package com.baeldung.twitterhdfs.sink;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.stream.annotation.EnableBinding;
import org.springframework.cloud.stream.app.hdfs.sink.HdfsSinkConfiguration;
import org.springframework.cloud.stream.messaging.Sink;
import org.springframework.context.annotation.Import;
import org.springframework.integration.annotation.ServiceActivator;
@SpringBootApplication
@EnableBinding(Sink.class)
@Import(HdfsSinkConfiguration.class)
public class SinkApp {
Logger log = LoggerFactory.getLogger(SinkApp.class);
@ServiceActivator(inputChannel= Sink.INPUT)
public void loggerSink(Object payload) {
log.info("Received: " + payload);
}
}

26
spring-cloud/spring-cloud-stream-starters/boot/src/main/java/com/baeldung/twitterhdfs/source/SourceApp.java Normal file
View File

@ -0,0 +1,26 @@
package com.baeldung.twitterhdfs.source;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.stream.annotation.EnableBinding;
import org.springframework.cloud.stream.app.twitterstream.source.TwitterstreamSourceConfiguration;
import org.springframework.cloud.stream.messaging.Source;
import org.springframework.context.annotation.Import;
import org.springframework.integration.annotation.InboundChannelAdapter;
import java.text.SimpleDateFormat;
import java.util.Date;
@SpringBootApplication
@EnableBinding(Source.class)
@Import(TwitterstreamSourceConfiguration.class)
public class SourceApp {
Logger log = LoggerFactory.getLogger(SourceApp.class);
@InboundChannelAdapter(value = Source.OUTPUT)
public String timerMessageSource() {
return new SimpleDateFormat().format(new Date());
}
}

6
spring-cloud/spring-cloud-stream-starters/boot/src/main/resources/application.properties Normal file
View File

@ -0,0 +1,6 @@
hdfs.fs-uri=hdfs://127.0.0.1:50010/
twitter.credentials.access-token=
twitter.credentials.access-token-secret=
twitter.credentials.consumer-key=
twitter.credentials.consumer-secret=

2
spring-cloud/spring-cloud-stream-starters/hdfs/hdfs.sh
View File

@ -5,7 +5,7 @@ git clone https://github.com/spring-cloud-stream-app-starters/hdfs.git
# Build it
./mvnw clean install -PgenerateApps
# RUn it
# Run it
cd apps
# Optionally inject application.properties prior to build
java -jar hdfs-sink.jar --fsUri=http://osboxes:50075

8
spring-cloud/spring-cloud-stream-starters/twitter/application.properties
View File

@ -1,4 +1,4 @@
twitter.credentials.access-token=932486336086286336-2HURQbA2cYzX5hixgAshIBy2Dhefupn
twitter.credentials.access-token-secret=0pyZ7etHvro8x85QSXxsqYFzYk63bK6DS5nNYy0R3l1io
twitter.credentials.consumer-key=10xCXkRYi5xLFYq3P0ymWGEwJ
twitter.credentials.consumer-secret=VfyCUcGfAQ2aWcd3uTg8GmVGyhUfAcNJU6ksG09TAtPMqhmWTS
twitter.credentials.access-token=
twitter.credentials.access-token-secret=
twitter.credentials.consumer-key=
twitter.credentials.consumer-secret=

2
spring-cloud/spring-cloud-stream-starters/twitter/twitter.sh
View File

@ -5,7 +5,7 @@ git clone https://github.com/spring-cloud-stream-app-starters/twitter.git
# Build it
./mvnw clean install -PgenerateApps
# RUn it
# Run it
cd apps
# Optionally inject application.properties prior to build
java -jar twitter_stream_source.jar --consumerKey=<CONSUMER_KEY> --consumerSecret=<CONSUMER_SECRET> \