iSharkFly-Docs
/
java-tutorials
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

New Article - ETL with Spring Cloud Data Flow (#5388) 

* Including the ETL files to the repository

* Rename READMET.md to README.MD

Renamed to correct README.

* Update README.MD

Additional line brakes

* Formatting

* Removing unecessary autogenerated files
This commit is contained in:
Norberto Ritzmann Júnior 2018-10-06 00:09:13 -03:00 committed by Grzegorz Piwowarek
parent a1999542c1
commit cb6e84c59e
14 changed files with 296 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
spring-cloud-data-flow/etl/README.MD Normal file
View File

@ -0,0 +1,9 @@
# Overview
This is an example of a ETL stream pipeline, mixing a starter application with custom transform and sink.
# Applications
JDBC Source - Application Starter distributed by default
customer-transform - Custom application to transform the data
customer-mongodb-sink - Custom application to sink the data

75
spring-cloud-data-flow/etl/customer-mongodb-sink/pom.xml Normal file
View File

@ -0,0 +1,75 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.customer</groupId>
<artifactId>customer-mongodb-sink</artifactId>
<packaging>jar</packaging>
<name>customer-mongodb-sink</name>
<description>Example ETL Load Project</description>
<parent>
<artifactId>parent-boot-2</artifactId>
<groupId>com.baeldung</groupId>
<version>0.0.1-SNAPSHOT</version>
<relativePath>../../../parent-boot-2</relativePath>
</parent>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>1.8</java.version>
<spring-cloud.version>Finchley.SR1</spring-cloud.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-stream</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-stream-binder-rabbit</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-mongodb</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-stream-test-support</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-dependencies</artifactId>
<version>${spring-cloud.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

27
spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/Customer.java Normal file
View File

@ -0,0 +1,27 @@
package com.customer.customermongodbsink;
import org.springframework.data.mongodb.core.mapping.Document;
@Document(collection = "customer")
public class Customer {
private Long id;
private String name;
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
}

18
spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/CustomerListener.java Normal file
View File

@ -0,0 +1,18 @@
package com.customer.customermongodbsink;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cloud.stream.annotation.EnableBinding;
import org.springframework.cloud.stream.annotation.StreamListener;
import org.springframework.cloud.stream.messaging.Sink;
@EnableBinding(Sink.class)
public class CustomerListener {
@Autowired
private CustomerRepository repository;
@StreamListener(Sink.INPUT)
public void save(Customer customer) {
repository.save(customer);
}
}

12
spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/CustomerMongodbSinkApplication.java Normal file
View File

@ -0,0 +1,12 @@
package com.customer.customermongodbsink;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class CustomerMongodbSinkApplication {
public static void main(String[] args) {
SpringApplication.run(CustomerMongodbSinkApplication.class, args);
}
}

9
spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/CustomerRepository.java Normal file
View File

@ -0,0 +1,9 @@
package com.customer.customermongodbsink;
import org.springframework.data.mongodb.repository.MongoRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface CustomerRepository extends MongoRepository<Customer, Long> {
}

0
spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/resources/application.properties Normal file
View File

68
spring-cloud-data-flow/etl/customer-transform/pom.xml Normal file
View File

@ -0,0 +1,68 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.customer</groupId>
<artifactId>customer-transform</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<name>customer-transform</name>
<description>Example transform ETL step</description>
<parent>
<artifactId>parent-boot-2</artifactId>
<groupId>com.baeldung</groupId>
<version>0.0.1-SNAPSHOT</version>
<relativePath>../../../parent-boot-2</relativePath>
</parent>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>1.8</java.version>
<spring-cloud.version>Finchley.SR1</spring-cloud.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-stream-binder-rabbit</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-stream-test-support</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-dependencies</artifactId>
<version>${spring-cloud.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

29
spring-cloud-data-flow/etl/customer-transform/src/main/java/com/customer/customertransform/Customer.java Normal file
View File

@ -0,0 +1,29 @@
package com.customer.customertransform;
import com.fasterxml.jackson.annotation.JsonProperty;
public class Customer {
private Long id;
private String name;
@JsonProperty("customer_name")
public void setName(String name) {
this.name = name;
}
@JsonProperty("name")
public String getName() {
return name;
}
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
}

16
spring-cloud-data-flow/etl/customer-transform/src/main/java/com/customer/customertransform/CustomerProcessorConfiguration.java Normal file
View File

@ -0,0 +1,16 @@
package com.customer.customertransform;
import org.springframework.cloud.stream.annotation.EnableBinding;
import org.springframework.cloud.stream.messaging.Processor;
import org.springframework.integration.annotation.Transformer;
@EnableBinding(Processor.class)
public class CustomerProcessorConfiguration {
@Transformer(inputChannel = Processor.INPUT, outputChannel = Processor.OUTPUT)
public Customer convertToPojo(Customer payload) {
return payload;
}
}

12
spring-cloud-data-flow/etl/customer-transform/src/main/java/com/customer/customertransform/CustomerTransformApplication.java Normal file
View File

@ -0,0 +1,12 @@
package com.customer.customertransform;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class CustomerTransformApplication {
public static void main(String[] args) {
SpringApplication.run(CustomerTransformApplication.class, args);
}
}

0
spring-cloud-data-flow/etl/customer-transform/src/main/resources/application.properties Normal file
View File

20
spring-cloud-data-flow/etl/pom.xml Normal file
View File

@ -0,0 +1,20 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.baeldung.spring.cloud</groupId>
<artifactId>etl-spring-cloud-data-flow</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>pom</packaging>
<parent>
<groupId>org.baeldung.spring.cloud</groupId>
<artifactId>spring-cloud-data-flow</artifactId>
<version>0.0.1-SNAPSHOT</version>
</parent>
<modules>
<module>customer-mongodb-sink</module>
<module>customer-transform</module>
</modules>
</project>

1
spring-cloud-data-flow/pom.xml
View File

@ -19,6 +19,7 @@
<module>time-processor</module>
<module>log-sink</module>
<module>batch-job</module>
<module>etl</module>
</modules>
</project>