BAEL-1175 - Finalized example and improved scripts

spring-cloud/spring-cloud-stream-starters/bash/hadoop.sh

@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 # For Ubuntu 14.04
 # Inspired from: https://github.com/curran/setupHadoop/blob/master/setupHadoop.sh
 # Use from the user directory
@@ -32,7 +34,7 @@ sudo mv ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys
 
 # SSH
 ssh localhost
-# Authenticate with local user
+# authenticate with osboxes.org
 
 # Start NameNode daemon and DataNode daemon
 start-dfs.sh
@@ -40,3 +42,5 @@ start-dfs.sh
 
 # Install Maven
 sudo apt-get install maven
+
+# Access Hadoop - http://localhost:50070

spring-cloud/spring-cloud-stream-starters/boot/.gitignore

@@ -0,0 +1,3 @@
+.idea
+*/target/*
+*.iml

spring-cloud/spring-cloud-stream-starters/boot/pom.xml

@@ -0,0 +1,55 @@
+<?xml version="1.0"?>
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+         xmlns="http://maven.apache.org/POM/4.0.0">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.baeldung.twitterhdfs</groupId>
+    <artifactId>twitterhdfs</artifactId>
+    <packaging>jar</packaging>
+    <version>1.0.0</version>
+
+    <name>twitterhdfs</name>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+        <java.version>1.8</java.version>
+    </properties>
+
+    <parent>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-parent</artifactId>
+        <version>1.5.8.RELEASE</version>
+    </parent>
+
+    <dependencies>
+        <!-- Spring Stream Starter Apps -->
+        <dependency>
+            <groupId>org.springframework.cloud.stream.app</groupId>
+            <artifactId>spring-cloud-starter-stream-source-twitterstream</artifactId>
+            <version>1.3.1.RELEASE</version>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.cloud.stream.app</groupId>
+            <artifactId>spring-cloud-starter-stream-sink-hdfs</artifactId>
+            <version>1.3.1.RELEASE</version>
+        </dependency>
+
+        <!-- JSTL/JSP -->
+        <dependency>
+            <groupId>javax.servlet</groupId>
+            <artifactId>jstl</artifactId>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+        <finalName>twitterhdfs</finalName>
+    </build>
+</project>

spring-cloud/spring-cloud-stream-starters/boot/src/main/java/com/baeldung/twitterhdfs/aggregate/AggregateApp.java

@@ -0,0 +1,18 @@
+package com.baeldung.twitterhdfs.aggregate;
+
+import com.baeldung.twitterhdfs.processor.ProcessorApp;
+import com.baeldung.twitterhdfs.source.SourceApp;
+import com.baeldung.twitterhdfs.sink.SinkApp;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.cloud.stream.aggregate.AggregateApplicationBuilder;
+
+@SpringBootApplication
+public class AggregateApp {
+    public static void main(String[] args) {
+        new AggregateApplicationBuilder()
+                .from(SourceApp.class).args("--fixedDelay=5000")
+                .via(ProcessorApp.class)
+                .to(SinkApp.class).args("--debug=true")
+                .run(args);
+    }
+}

spring-cloud/spring-cloud-stream-starters/boot/src/main/java/com/baeldung/twitterhdfs/processor/ProcessorApp.java

@@ -0,0 +1,20 @@
+package com.baeldung.twitterhdfs.processor;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.cloud.stream.annotation.EnableBinding;
+import org.springframework.cloud.stream.messaging.Processor;
+import org.springframework.integration.annotation.Transformer;
+
+@SpringBootApplication
+@EnableBinding(Processor.class)
+public class ProcessorApp {
+    Logger log = LoggerFactory.getLogger(ProcessorApp.class);
+
+    @Transformer(inputChannel = Processor.INPUT, outputChannel = Processor.OUTPUT)
+    public String processMessage(String payload) {
+        log.info("Payload received!");
+        return payload;
+    }
+}

spring-cloud/spring-cloud-stream-starters/boot/src/main/java/com/baeldung/twitterhdfs/sink/SinkApp.java

@@ -0,0 +1,22 @@
+package com.baeldung.twitterhdfs.sink;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.cloud.stream.annotation.EnableBinding;
+import org.springframework.cloud.stream.app.hdfs.sink.HdfsSinkConfiguration;
+import org.springframework.cloud.stream.messaging.Sink;
+import org.springframework.context.annotation.Import;
+import org.springframework.integration.annotation.ServiceActivator;
+
+@SpringBootApplication
+@EnableBinding(Sink.class)
+@Import(HdfsSinkConfiguration.class)
+public class SinkApp {
+    Logger log = LoggerFactory.getLogger(SinkApp.class);
+
+    @ServiceActivator(inputChannel= Sink.INPUT)
+    public void loggerSink(Object payload) {
+        log.info("Received: " + payload);
+    }
+}

spring-cloud/spring-cloud-stream-starters/boot/src/main/java/com/baeldung/twitterhdfs/source/SourceApp.java

@@ -0,0 +1,26 @@
+package com.baeldung.twitterhdfs.source;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.cloud.stream.annotation.EnableBinding;
+import org.springframework.cloud.stream.app.twitterstream.source.TwitterstreamSourceConfiguration;
+import org.springframework.cloud.stream.messaging.Source;
+import org.springframework.context.annotation.Import;
+import org.springframework.integration.annotation.InboundChannelAdapter;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+@SpringBootApplication
+@EnableBinding(Source.class)
+@Import(TwitterstreamSourceConfiguration.class)
+public class SourceApp {
+    Logger log = LoggerFactory.getLogger(SourceApp.class);
+
+    @InboundChannelAdapter(value = Source.OUTPUT)
+    public String timerMessageSource() {
+        return new SimpleDateFormat().format(new Date());
+    }
+
+}

spring-cloud/spring-cloud-stream-starters/boot/src/main/resources/application.properties

@@ -0,0 +1,6 @@
+hdfs.fs-uri=hdfs://127.0.0.1:50010/
+
+twitter.credentials.access-token=
+twitter.credentials.access-token-secret=
+twitter.credentials.consumer-key=
+twitter.credentials.consumer-secret=

spring-cloud/spring-cloud-stream-starters/hdfs/hdfs.sh

@@ -5,7 +5,7 @@ git clone https://github.com/spring-cloud-stream-app-starters/hdfs.git
 # Build it
 ./mvnw clean install -PgenerateApps
 
-# RUn it
+# Run it
 cd apps
 # Optionally inject application.properties prior to build
 java -jar hdfs-sink.jar --fsUri=http://osboxes:50075

spring-cloud/spring-cloud-stream-starters/twitter/application.properties

@@ -1,4 +1,4 @@
-twitter.credentials.access-token=932486336086286336-2HURQbA2cYzX5hixgAshIBy2Dhefupn
+twitter.credentials.access-token=
-twitter.credentials.access-token-secret=0pyZ7etHvro8x85QSXxsqYFzYk63bK6DS5nNYy0R3l1io
+twitter.credentials.access-token-secret=
-twitter.credentials.consumer-key=10xCXkRYi5xLFYq3P0ymWGEwJ
+twitter.credentials.consumer-key=
-twitter.credentials.consumer-secret=VfyCUcGfAQ2aWcd3uTg8GmVGyhUfAcNJU6ksG09TAtPMqhmWTS
+twitter.credentials.consumer-secret=

spring-cloud/spring-cloud-stream-starters/twitter/twitter.sh

@@ -5,7 +5,7 @@ git clone https://github.com/spring-cloud-stream-app-starters/twitter.git
 # Build it
 ./mvnw clean install -PgenerateApps
 
-# RUn it
+# Run it
 cd apps
 # Optionally inject application.properties prior to build
 java -jar twitter_stream_source.jar --consumerKey=<CONSUMER_KEY> --consumerSecret=<CONSUMER_SECRET> \