From 5b56b1ed392e3448f940601dd74367b88577a0d1 Mon Sep 17 00:00:00 2001 From: Ulisses Lima Date: Fri, 11 Mar 2022 20:11:54 -0300 Subject: [PATCH] BAEL-5403 - Import Data to MongoDB from Json File using Java * Project * Integration tests --- .../SpringBootPersistenceApplication.java | 48 +++++++- .../boot/json/convertfile/ImportUtils.java | 60 ++++++++++ .../json/convertfile/dao/BookRepository.java | 9 ++ .../boot/json/convertfile/data/Book.java | 38 +++++++ .../service/ImportJsonService.java | 74 ++++++++++++ .../json/convertfile/web/BookController.java | 51 +++++++++ .../convertfile/web/ImportJsonController.java | 35 ++++++ .../boot.json.convertfile/data.json.log | 3 + .../ImportJsonServiceIntegrationTest.java | 105 ++++++++++++++++++ .../boot.json.convertfile/books.json.log | 2 + .../boot.json.convertfile/movies.json.log | 3 + 11 files changed, 427 insertions(+), 1 deletion(-) create mode 100644 persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/ImportUtils.java create mode 100644 persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/dao/BookRepository.java create mode 100644 persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/data/Book.java create mode 100644 persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/service/ImportJsonService.java create mode 100644 persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/web/BookController.java create mode 100644 persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/web/ImportJsonController.java create mode 100644 persistence-modules/spring-boot-persistence-mongodb/src/main/resources/boot.json.convertfile/data.json.log create mode 100644 persistence-modules/spring-boot-persistence-mongodb/src/test/java/com/baeldung/boot/json/convertfile/service/ImportJsonServiceIntegrationTest.java create mode 100644 persistence-modules/spring-boot-persistence-mongodb/src/test/resources/boot.json.convertfile/books.json.log create mode 100644 persistence-modules/spring-boot-persistence-mongodb/src/test/resources/boot.json.convertfile/movies.json.log diff --git a/persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/SpringBootPersistenceApplication.java b/persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/SpringBootPersistenceApplication.java index 2dff3f37df..a5cb443291 100644 --- a/persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/SpringBootPersistenceApplication.java +++ b/persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/SpringBootPersistenceApplication.java @@ -1,13 +1,59 @@ package com.baeldung; +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.ApplicationContext; + +import com.baeldung.boot.json.convertfile.ImportUtils; +import com.baeldung.boot.json.convertfile.service.ImportJsonService; @SpringBootApplication -public class SpringBootPersistenceApplication { +public class SpringBootPersistenceApplication implements ApplicationRunner { + private Logger log = LogManager.getLogger(this.getClass()); + private static final String RESOURCE_PREFIX = "classpath:"; + + @Autowired + private ApplicationContext context; public static void main(String ... args) { SpringApplication.run(SpringBootPersistenceApplication.class, args); } + @Override + public void run(ApplicationArguments args) throws Exception { + if (args.containsOption("import")) { + if (!args.containsOption("collection")) + throw new IllegalArgumentException("required option: --collection with collection name when using --import"); + + String collection = args.getOptionValues("collection") + .get(0); + + List sources = args.getOptionValues("import"); + for (String source : sources) { + List jsonLines = new ArrayList<>(); + if (source.startsWith(RESOURCE_PREFIX)) { + String resource = source.substring(RESOURCE_PREFIX.length()); + jsonLines = ImportUtils.linesFromResource(resource); + } else { + jsonLines = ImportUtils.lines(new File(source)); + } + + if (jsonLines == null || jsonLines.isEmpty()) + throw new IllegalArgumentException("no input to import"); + + ImportJsonService importService = context.getBean(ImportJsonService.class); + String result = importService.importTo(collection, jsonLines); + log.info(source + " - result: " + result); + } + } + } } diff --git a/persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/ImportUtils.java b/persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/ImportUtils.java new file mode 100644 index 0000000000..c09746a33d --- /dev/null +++ b/persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/ImportUtils.java @@ -0,0 +1,60 @@ +package com.baeldung.boot.json.convertfile; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; +import org.springframework.web.multipart.MultipartFile; + +public class ImportUtils { + private static Logger log = LogManager.getLogger(ImportUtils.class); + + public static List lines(String json) { + if (json == null) + return Collections.emptyList(); + + String[] split = json.split("[\\r\\n]+"); + return Arrays.asList(split); + } + + public static List lines(MultipartFile file) { + try { + Path tmp = Files.write(File.createTempFile("mongo-upload", null) + .toPath(), file.getBytes()); + + return Files.readAllLines(tmp); + } catch (IOException e) { + log.error("reading lines from " + file, e); + return null; + } + } + + public static List lines(File file) { + try { + return Files.readAllLines(file.toPath()); + } catch (IOException e) { + log.error("reading lines from " + file, e); + return null; + } + } + + public static List linesFromResource(String resource) { + Resource input = new ClassPathResource(resource); + try { + Path path = input.getFile() + .toPath(); + return Files.readAllLines(path); + } catch (IOException e) { + log.error("reading lines from classpath resource: " + resource, e); + return null; + } + } +} diff --git a/persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/dao/BookRepository.java b/persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/dao/BookRepository.java new file mode 100644 index 0000000000..523be429da --- /dev/null +++ b/persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/dao/BookRepository.java @@ -0,0 +1,9 @@ +package com.baeldung.boot.json.convertfile.dao; + +import org.springframework.data.mongodb.repository.MongoRepository; + +import com.baeldung.boot.json.convertfile.data.Book; + +public interface BookRepository extends MongoRepository { + +} diff --git a/persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/data/Book.java b/persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/data/Book.java new file mode 100644 index 0000000000..c352cdeecd --- /dev/null +++ b/persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/data/Book.java @@ -0,0 +1,38 @@ +package com.baeldung.boot.json.convertfile.data; + +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.Document; + +@Document("books") +public class Book { + @Id + private String id; + + private String name; + + private String genre; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getGenre() { + return genre; + } + + public void setGenre(String genre) { + this.genre = genre; + } +} diff --git a/persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/service/ImportJsonService.java b/persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/service/ImportJsonService.java new file mode 100644 index 0000000000..4800a85da3 --- /dev/null +++ b/persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/service/ImportJsonService.java @@ -0,0 +1,74 @@ +package com.baeldung.boot.json.convertfile.service; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bson.Document; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.mongodb.MongoBulkWriteException; + +@Service +public class ImportJsonService { + private Logger log = LogManager.getLogger(this.getClass()); + + @Autowired + private MongoTemplate mongo; + + public String importTo(Class type, List jsonLines) { + List mongoDocs = generateMongoDocs(jsonLines, type); + String collection = type.getAnnotation(org.springframework.data.mongodb.core.mapping.Document.class) + .value(); + int inserts = insertInto(collection, mongoDocs); + return inserts + "/" + jsonLines.size(); + } + + public String importTo(String collection, List jsonLines) { + List mongoDocs = generateMongoDocs(jsonLines); + int inserts = insertInto(collection, mongoDocs); + return inserts + "/" + jsonLines.size(); + } + + private int insertInto(String collection, List mongoDocs) { + try { + Collection inserts = mongo.insert(mongoDocs, collection); + return inserts.size(); + } catch (DataIntegrityViolationException e) { + log.error("importing docs", e); + if (e.getCause() instanceof MongoBulkWriteException) { + return ((MongoBulkWriteException) e.getCause()).getWriteResult() + .getInsertedCount(); + } + return 0; + } + } + + private List generateMongoDocs(List lines) { + return generateMongoDocs(lines, null); + } + + private List generateMongoDocs(List lines, Class type) { + ObjectMapper mapper = new ObjectMapper(); + + List docs = new ArrayList<>(); + for (String json : lines) { + try { + if (type != null) { + T v = mapper.readValue(json, type); + json = mapper.writeValueAsString(v); + } + docs.add(Document.parse(json)); + } catch (Throwable e) { + log.error("parsing: " + json, e); + } + } + return docs; + } +} diff --git a/persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/web/BookController.java b/persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/web/BookController.java new file mode 100644 index 0000000000..3fbe1bb34c --- /dev/null +++ b/persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/web/BookController.java @@ -0,0 +1,51 @@ +package com.baeldung.boot.json.convertfile.web; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestPart; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import com.baeldung.boot.json.convertfile.ImportUtils; +import com.baeldung.boot.json.convertfile.dao.BookRepository; +import com.baeldung.boot.json.convertfile.data.Book; +import com.baeldung.boot.json.convertfile.service.ImportJsonService; + +@RestController +@RequestMapping("/books") +public class BookController { + @Autowired + private BookRepository books; + + @Autowired + private ImportJsonService service; + + @PostMapping + public Book postBook(@RequestBody Book book) throws IOException { + return books.insert(book); + } + + @GetMapping + public List getBooks() { + return books.findAll(); + } + + @GetMapping("/{id}") + public Optional getBook(@PathVariable String id) { + return books.findById(id); + } + + @PostMapping("/import/file") + public String postJsonFile(@RequestPart("parts") MultipartFile jsonStringsFile) throws IOException { + List jsonLines = ImportUtils.lines(jsonStringsFile); + return service.importTo(Book.class, jsonLines); + } +} diff --git a/persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/web/ImportJsonController.java b/persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/web/ImportJsonController.java new file mode 100644 index 0000000000..2e475c9e8c --- /dev/null +++ b/persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/web/ImportJsonController.java @@ -0,0 +1,35 @@ +package com.baeldung.boot.json.convertfile.web; + +import java.io.IOException; +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestPart; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import com.baeldung.boot.json.convertfile.ImportUtils; +import com.baeldung.boot.json.convertfile.service.ImportJsonService; + +@RestController +@RequestMapping("/import-json") +public class ImportJsonController { + @Autowired + private ImportJsonService service; + + @PostMapping("/{collection}") + public String postJson(@RequestBody String jsonStrings, @PathVariable String collection) { + List jsonLines = ImportUtils.lines(jsonStrings); + return service.importTo(collection, jsonLines); + } + + @PostMapping("/file/{collection}") + public String postJsonFile(@RequestPart("parts") MultipartFile jsonStringsFile, @PathVariable String collection) throws IOException { + List jsonLines = ImportUtils.lines(jsonStringsFile); + return service.importTo(collection, jsonLines); + } +} diff --git a/persistence-modules/spring-boot-persistence-mongodb/src/main/resources/boot.json.convertfile/data.json.log b/persistence-modules/spring-boot-persistence-mongodb/src/main/resources/boot.json.convertfile/data.json.log new file mode 100644 index 0000000000..4c1a23f435 --- /dev/null +++ b/persistence-modules/spring-boot-persistence-mongodb/src/main/resources/boot.json.convertfile/data.json.log @@ -0,0 +1,3 @@ +{"name":"Book A", "genre": "Comedy"} +{"name":"Book B", "genre": "Thriller"} +{"_id": "mongo-id", "name":"Book C", "genre": "Drama"} diff --git a/persistence-modules/spring-boot-persistence-mongodb/src/test/java/com/baeldung/boot/json/convertfile/service/ImportJsonServiceIntegrationTest.java b/persistence-modules/spring-boot-persistence-mongodb/src/test/java/com/baeldung/boot/json/convertfile/service/ImportJsonServiceIntegrationTest.java new file mode 100644 index 0000000000..611cdffa88 --- /dev/null +++ b/persistence-modules/spring-boot-persistence-mongodb/src/test/java/com/baeldung/boot/json/convertfile/service/ImportJsonServiceIntegrationTest.java @@ -0,0 +1,105 @@ +package com.baeldung.boot.json.convertfile.service; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.junit4.SpringRunner; + +import com.baeldung.boot.json.convertfile.ImportUtils; +import com.baeldung.boot.json.convertfile.dao.BookRepository; +import com.baeldung.boot.json.convertfile.data.Book; +import com.mongodb.DBObject; + +@SpringBootTest +@DirtiesContext +@RunWith(SpringRunner.class) +public class ImportJsonServiceIntegrationTest { + @Autowired + private ImportJsonService service; + + @Autowired + private MongoTemplate mongoDb; + + @Autowired + BookRepository bookRepository; + + @Test + public void givenJsonString_whenGenericType_thenDocumentImported() { + String collection = "items"; + List docs = mongoDb.findAll(DBObject.class, collection); + int sizeBefore = docs.size(); + + String json = "{\"name\":\"Item A\"}\n{\"name\":\"Item B\"}"; + List jsonLines = ImportUtils.lines(json); + service.importTo(collection, jsonLines); + + docs = mongoDb.findAll(DBObject.class, collection); + int sizeAfter = docs.size(); + assertThat(sizeAfter - sizeBefore).isEqualTo(jsonLines.size()); + } + + @Test + public void givenJsonFile_whenClasspathSource_thenDocumentImported() { + String collection = "movies"; + List docs = mongoDb.findAll(DBObject.class, collection); + int sizeBefore = docs.size(); + + String resource = "boot.json.convertfile/movies.json.log"; + List jsonLines = ImportUtils.linesFromResource(resource); + service.importTo(collection, jsonLines); + + docs = mongoDb.findAll(DBObject.class, collection); + int sizeAfter = docs.size(); + assertThat(sizeAfter - sizeBefore).isEqualTo(jsonLines.size()); + } + + @Test + public void givenJsonClasspathFile_whenCorrectlyTyped_thenDocumentImported() { + List books = bookRepository.findAll(); + int sizeBefore = books.size(); + + String resource = "boot.json.convertfile/books.json.log"; + List jsonLines = ImportUtils.linesFromResource(resource); + service.importTo(Book.class, jsonLines); + + books = bookRepository.findAll(); + int sizeAfter = books.size(); + assertThat(sizeAfter - sizeBefore).isEqualTo(jsonLines.size()); + } + + @Test + public void givenIncorrectlyTypedJson_whenUsingTypes_thenDocumentNotImported() { + List books = bookRepository.findAll(); + int sizeBefore = books.size(); + + String resource = "boot.json.convertfile/movies.json.log"; + List jsonLines = ImportUtils.linesFromResource(resource); + service.importTo(Book.class, jsonLines); + + books = bookRepository.findAll(); + int sizeAfter = books.size(); + assertThat(sizeAfter - sizeBefore).isEqualTo(0); + } + + @Test + public void whenInvalidJson_thenDocumentNotImported() { + String collection = "items"; + List docs = mongoDb.findAll(DBObject.class, collection); + int sizeBefore = docs.size(); + + String json = "{name: Item A}\n{name: Item B}"; + List jsonLines = ImportUtils.lines(json); + service.importTo(collection, jsonLines); + + docs = mongoDb.findAll(DBObject.class, collection); + int sizeAfter = docs.size(); + assertThat(sizeAfter - sizeBefore).isEqualTo(0); + } +} diff --git a/persistence-modules/spring-boot-persistence-mongodb/src/test/resources/boot.json.convertfile/books.json.log b/persistence-modules/spring-boot-persistence-mongodb/src/test/resources/boot.json.convertfile/books.json.log new file mode 100644 index 0000000000..63c618ba82 --- /dev/null +++ b/persistence-modules/spring-boot-persistence-mongodb/src/test/resources/boot.json.convertfile/books.json.log @@ -0,0 +1,2 @@ +{"name":"Movie A", "genre": "Comedy"} +{"name":"Movie B", "genre": "Thriller"} diff --git a/persistence-modules/spring-boot-persistence-mongodb/src/test/resources/boot.json.convertfile/movies.json.log b/persistence-modules/spring-boot-persistence-mongodb/src/test/resources/boot.json.convertfile/movies.json.log new file mode 100644 index 0000000000..7658d8610e --- /dev/null +++ b/persistence-modules/spring-boot-persistence-mongodb/src/test/resources/boot.json.convertfile/movies.json.log @@ -0,0 +1,3 @@ +{"title":"Movie A", "genre": "Comedy"} +{"title":"Movie B", "genre": "Thriller"} +{"_id": "mongo-id", "title":"Movie C", "genre": "Drama"}