New Article - ETL with Spring Cloud Data Flow (#5388)

* Including the ETL files to the repository

* Rename READMET.md to README.MD

Renamed to correct README.

* Update README.MD

Additional line brakes

* Formatting

* Removing unecessary autogenerated files

spring-cloud-data-flow/etl/README.MD

@@ -0,0 +1,9 @@
+# Overview
+This is an example of a ETL stream pipeline, mixing a starter application with custom transform and sink.
+
+# Applications
+JDBC Source - Application Starter distributed by default
+
+customer-transform - Custom application to transform the data
+
+customer-mongodb-sink - Custom application to sink the data

spring-cloud-data-flow/etl/customer-mongodb-sink/pom.xml

@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.customer</groupId>
+    <artifactId>customer-mongodb-sink</artifactId>
+    <packaging>jar</packaging>
+
+    <name>customer-mongodb-sink</name>
+    <description>Example ETL Load Project</description>
+
+    <parent>
+        <artifactId>parent-boot-2</artifactId>
+        <groupId>com.baeldung</groupId>
+        <version>0.0.1-SNAPSHOT</version>
+        <relativePath>../../../parent-boot-2</relativePath>
+    </parent>
+ 
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+        <java.version>1.8</java.version>
+        <spring-cloud.version>Finchley.SR1</spring-cloud.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-stream</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-stream-binder-rabbit</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-mongodb</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-stream-test-support</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>org.springframework.cloud</groupId>
+                <artifactId>spring-cloud-dependencies</artifactId>
+                <version>${spring-cloud.version}</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+
+</project>

spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/Customer.java

@@ -0,0 +1,27 @@
+package com.customer.customermongodbsink;
+
+import org.springframework.data.mongodb.core.mapping.Document;
+
+@Document(collection = "customer")
+public class Customer {
+
+    private Long id;
+    private String name;
+
+    public Long getId() {
+        return id;
+    }
+
+    public void setId(Long id) {
+        this.id = id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+}

spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/CustomerListener.java

@@ -0,0 +1,18 @@
+package com.customer.customermongodbsink;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.cloud.stream.annotation.EnableBinding;
+import org.springframework.cloud.stream.annotation.StreamListener;
+import org.springframework.cloud.stream.messaging.Sink;
+
+@EnableBinding(Sink.class)
+public class CustomerListener {
+
+    @Autowired
+    private CustomerRepository repository;
+
+    @StreamListener(Sink.INPUT)
+    public void save(Customer customer) {
+        repository.save(customer);
+    }
+}

spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/CustomerMongodbSinkApplication.java

@@ -0,0 +1,12 @@
+package com.customer.customermongodbsink;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class CustomerMongodbSinkApplication {
+
+    public static void main(String[] args) {
+        SpringApplication.run(CustomerMongodbSinkApplication.class, args);
+    }
+}

spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/CustomerRepository.java

@@ -0,0 +1,9 @@
+package com.customer.customermongodbsink;
+
+import org.springframework.data.mongodb.repository.MongoRepository;
+import org.springframework.stereotype.Repository;
+
+@Repository
+public interface CustomerRepository extends MongoRepository<Customer, Long> {
+
+}

spring-cloud-data-flow/etl/customer-transform/pom.xml

@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.customer</groupId>
+    <artifactId>customer-transform</artifactId>
+    <version>0.0.1-SNAPSHOT</version>
+    <packaging>jar</packaging>
+
+    <name>customer-transform</name>
+    <description>Example transform ETL step</description>
+
+    <parent>
+        <artifactId>parent-boot-2</artifactId>
+        <groupId>com.baeldung</groupId>
+        <version>0.0.1-SNAPSHOT</version>
+        <relativePath>../../../parent-boot-2</relativePath>
+    </parent>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+        <java.version>1.8</java.version>
+        <spring-cloud.version>Finchley.SR1</spring-cloud.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-stream-binder-rabbit</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-stream-test-support</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>org.springframework.cloud</groupId>
+                <artifactId>spring-cloud-dependencies</artifactId>
+                <version>${spring-cloud.version}</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+
+</project>

spring-cloud-data-flow/etl/customer-transform/src/main/java/com/customer/customertransform/Customer.java

@@ -0,0 +1,29 @@
+package com.customer.customertransform;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class Customer {
+
+    private Long id;
+
+    private String name;
+
+    @JsonProperty("customer_name")
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    @JsonProperty("name")
+    public String getName() {
+        return name;
+    }
+
+    public Long getId() {
+        return id;
+    }
+
+    public void setId(Long id) {
+        this.id = id;
+    }
+
+}

spring-cloud-data-flow/etl/customer-transform/src/main/java/com/customer/customertransform/CustomerProcessorConfiguration.java

@@ -0,0 +1,16 @@
+package com.customer.customertransform;
+
+
+import org.springframework.cloud.stream.annotation.EnableBinding;
+import org.springframework.cloud.stream.messaging.Processor;
+import org.springframework.integration.annotation.Transformer;
+
+@EnableBinding(Processor.class)
+public class CustomerProcessorConfiguration {
+
+    @Transformer(inputChannel = Processor.INPUT, outputChannel = Processor.OUTPUT)
+    public Customer convertToPojo(Customer payload) {
+
+        return payload;
+    }
+}

spring-cloud-data-flow/etl/customer-transform/src/main/java/com/customer/customertransform/CustomerTransformApplication.java

@@ -0,0 +1,12 @@
+package com.customer.customertransform;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class CustomerTransformApplication {
+
+    public static void main(String[] args) {
+        SpringApplication.run(CustomerTransformApplication.class, args);
+    }
+}

spring-cloud-data-flow/etl/pom.xml

@@ -0,0 +1,20 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>org.baeldung.spring.cloud</groupId>
+    <artifactId>etl-spring-cloud-data-flow</artifactId>
+    <version>0.0.1-SNAPSHOT</version>
+    <packaging>pom</packaging>
+
+    <parent>
+        <groupId>org.baeldung.spring.cloud</groupId>
+        <artifactId>spring-cloud-data-flow</artifactId>
+        <version>0.0.1-SNAPSHOT</version>
+    </parent>
+
+    <modules>
+        <module>customer-mongodb-sink</module>
+        <module>customer-transform</module>
+    </modules>
+
+</project>

spring-cloud-data-flow/pom.xml

@@ -19,6 +19,7 @@
         <module>time-processor</module>
         <module>log-sink</module>
         <module>batch-job</module>
+        <module>etl</module>
     </modules>
 
 </project>