BAEL-5403 - Import Data to MongoDB from Json File using Java

* Project
* Integration tests

persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/SpringBootPersistenceApplication.java

@@ -1,13 +1,59 @@
 package com.baeldung;
 
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.ApplicationArguments;
+import org.springframework.boot.ApplicationRunner;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.ApplicationContext;
+
+import com.baeldung.boot.json.convertfile.ImportUtils;
+import com.baeldung.boot.json.convertfile.service.ImportJsonService;
 
 @SpringBootApplication
-public class SpringBootPersistenceApplication {
+public class SpringBootPersistenceApplication implements ApplicationRunner {
+    private Logger log = LogManager.getLogger(this.getClass());
+    private static final String RESOURCE_PREFIX = "classpath:";
+
+    @Autowired
+    private ApplicationContext context;
 
     public static void main(String ... args) {
         SpringApplication.run(SpringBootPersistenceApplication.class, args);
     }
 
+    @Override
+    public void run(ApplicationArguments args) throws Exception {
+        if (args.containsOption("import")) {
+            if (!args.containsOption("collection"))
+                throw new IllegalArgumentException("required option: --collection with collection name when using --import");
+
+            String collection = args.getOptionValues("collection")
+                .get(0);
+
+            List<String> sources = args.getOptionValues("import");
+            for (String source : sources) {
+                List<String> jsonLines = new ArrayList<>();
+                if (source.startsWith(RESOURCE_PREFIX)) {
+                    String resource = source.substring(RESOURCE_PREFIX.length());
+                    jsonLines = ImportUtils.linesFromResource(resource);
+                } else {
+                    jsonLines = ImportUtils.lines(new File(source));
+                }
+
+                if (jsonLines == null || jsonLines.isEmpty())
+                    throw new IllegalArgumentException("no input to import");
+
+                ImportJsonService importService = context.getBean(ImportJsonService.class);
+                String result = importService.importTo(collection, jsonLines);
+                log.info(source + " - result: " + result);
+            }
+        }
+    }
 }

persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/ImportUtils.java

@@ -0,0 +1,60 @@
+package com.baeldung.boot.json.convertfile;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+import org.springframework.web.multipart.MultipartFile;
+
+public class ImportUtils {
+    private static Logger log = LogManager.getLogger(ImportUtils.class);
+
+    public static List<String> lines(String json) {
+        if (json == null)
+            return Collections.emptyList();
+
+        String[] split = json.split("[\\r\\n]+");
+        return Arrays.asList(split);
+    }
+
+    public static List<String> lines(MultipartFile file) {
+        try {
+            Path tmp = Files.write(File.createTempFile("mongo-upload", null)
+                .toPath(), file.getBytes());
+
+            return Files.readAllLines(tmp);
+        } catch (IOException e) {
+            log.error("reading lines from " + file, e);
+            return null;
+        }
+    }
+
+    public static List<String> lines(File file) {
+        try {
+            return Files.readAllLines(file.toPath());
+        } catch (IOException e) {
+            log.error("reading lines from " + file, e);
+            return null;
+        }
+    }
+
+    public static List<String> linesFromResource(String resource) {
+        Resource input = new ClassPathResource(resource);
+        try {
+            Path path = input.getFile()
+                .toPath();
+            return Files.readAllLines(path);
+        } catch (IOException e) {
+            log.error("reading lines from classpath resource: " + resource, e);
+            return null;
+        }
+    }
+}

persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/dao/BookRepository.java

@@ -0,0 +1,9 @@
+package com.baeldung.boot.json.convertfile.dao;
+
+import org.springframework.data.mongodb.repository.MongoRepository;
+
+import com.baeldung.boot.json.convertfile.data.Book;
+
+public interface BookRepository extends MongoRepository<Book, String> {
+
+}

persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/data/Book.java

@@ -0,0 +1,38 @@
+package com.baeldung.boot.json.convertfile.data;
+
+import org.springframework.data.annotation.Id;
+import org.springframework.data.mongodb.core.mapping.Document;
+
+@Document("books")
+public class Book {
+    @Id
+    private String id;
+
+    private String name;
+
+    private String genre;
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getGenre() {
+        return genre;
+    }
+
+    public void setGenre(String genre) {
+        this.genre = genre;
+    }
+}

persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/service/ImportJsonService.java

@@ -0,0 +1,74 @@
+package com.baeldung.boot.json.convertfile.service;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.bson.Document;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.dao.DataIntegrityViolationException;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.mongodb.MongoBulkWriteException;
+
+@Service
+public class ImportJsonService {
+    private Logger log = LogManager.getLogger(this.getClass());
+
+    @Autowired
+    private MongoTemplate mongo;
+
+    public String importTo(Class<?> type, List<String> jsonLines) {
+        List<Document> mongoDocs = generateMongoDocs(jsonLines, type);
+        String collection = type.getAnnotation(org.springframework.data.mongodb.core.mapping.Document.class)
+            .value();
+        int inserts = insertInto(collection, mongoDocs);
+        return inserts + "/" + jsonLines.size();
+    }
+
+    public String importTo(String collection, List<String> jsonLines) {
+        List<Document> mongoDocs = generateMongoDocs(jsonLines);
+        int inserts = insertInto(collection, mongoDocs);
+        return inserts + "/" + jsonLines.size();
+    }
+
+    private int insertInto(String collection, List<Document> mongoDocs) {
+        try {
+            Collection<Document> inserts = mongo.insert(mongoDocs, collection);
+            return inserts.size();
+        } catch (DataIntegrityViolationException e) {
+            log.error("importing docs", e);
+            if (e.getCause() instanceof MongoBulkWriteException) {
+                return ((MongoBulkWriteException) e.getCause()).getWriteResult()
+                    .getInsertedCount();
+            }
+            return 0;
+        }
+    }
+
+    private List<Document> generateMongoDocs(List<String> lines) {
+        return generateMongoDocs(lines, null);
+    }
+
+    private <T> List<Document> generateMongoDocs(List<String> lines, Class<T> type) {
+        ObjectMapper mapper = new ObjectMapper();
+
+        List<Document> docs = new ArrayList<>();
+        for (String json : lines) {
+            try {
+                if (type != null) {
+                    T v = mapper.readValue(json, type);
+                    json = mapper.writeValueAsString(v);
+                }
+                docs.add(Document.parse(json));
+            } catch (Throwable e) {
+                log.error("parsing: " + json, e);
+            }
+        }
+        return docs;
+    }
+}

persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/web/BookController.java

@@ -0,0 +1,51 @@
+package com.baeldung.boot.json.convertfile.web;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Optional;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestPart;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.multipart.MultipartFile;
+
+import com.baeldung.boot.json.convertfile.ImportUtils;
+import com.baeldung.boot.json.convertfile.dao.BookRepository;
+import com.baeldung.boot.json.convertfile.data.Book;
+import com.baeldung.boot.json.convertfile.service.ImportJsonService;
+
+@RestController
+@RequestMapping("/books")
+public class BookController {
+    @Autowired
+    private BookRepository books;
+
+    @Autowired
+    private ImportJsonService service;
+
+    @PostMapping
+    public Book postBook(@RequestBody Book book) throws IOException {
+        return books.insert(book);
+    }
+
+    @GetMapping
+    public List<Book> getBooks() {
+        return books.findAll();
+    }
+
+    @GetMapping("/{id}")
+    public Optional<Book> getBook(@PathVariable String id) {
+        return books.findById(id);
+    }
+
+    @PostMapping("/import/file")
+    public String postJsonFile(@RequestPart("parts") MultipartFile jsonStringsFile) throws IOException {
+        List<String> jsonLines = ImportUtils.lines(jsonStringsFile);
+        return service.importTo(Book.class, jsonLines);
+    }
+}

persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/web/ImportJsonController.java

@@ -0,0 +1,35 @@
+package com.baeldung.boot.json.convertfile.web;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestPart;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.multipart.MultipartFile;
+
+import com.baeldung.boot.json.convertfile.ImportUtils;
+import com.baeldung.boot.json.convertfile.service.ImportJsonService;
+
+@RestController
+@RequestMapping("/import-json")
+public class ImportJsonController {
+    @Autowired
+    private ImportJsonService service;
+
+    @PostMapping("/{collection}")
+    public String postJson(@RequestBody String jsonStrings, @PathVariable String collection) {
+        List<String> jsonLines = ImportUtils.lines(jsonStrings);
+        return service.importTo(collection, jsonLines);
+    }
+
+    @PostMapping("/file/{collection}")
+    public String postJsonFile(@RequestPart("parts") MultipartFile jsonStringsFile, @PathVariable String collection) throws IOException {
+        List<String> jsonLines = ImportUtils.lines(jsonStringsFile);
+        return service.importTo(collection, jsonLines);
+    }
+}

persistence-modules/spring-boot-persistence-mongodb/src/main/resources/boot.json.convertfile/data.json.log

@@ -0,0 +1,3 @@
+{"name":"Book A", "genre": "Comedy"}
+{"name":"Book B", "genre": "Thriller"}
+{"_id": "mongo-id", "name":"Book C", "genre": "Drama"}

persistence-modules/spring-boot-persistence-mongodb/src/test/java/com/baeldung/boot/json/convertfile/service/ImportJsonServiceIntegrationTest.java

@@ -0,0 +1,105 @@
+package com.baeldung.boot.json.convertfile.service;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.List;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import com.baeldung.boot.json.convertfile.ImportUtils;
+import com.baeldung.boot.json.convertfile.dao.BookRepository;
+import com.baeldung.boot.json.convertfile.data.Book;
+import com.mongodb.DBObject;
+
+@SpringBootTest
+@DirtiesContext
+@RunWith(SpringRunner.class)
+public class ImportJsonServiceIntegrationTest {
+    @Autowired
+    private ImportJsonService service;
+
+    @Autowired
+    private MongoTemplate mongoDb;
+
+    @Autowired
+    BookRepository bookRepository;
+
+    @Test
+    public void givenJsonString_whenGenericType_thenDocumentImported() {
+        String collection = "items";
+        List<DBObject> docs = mongoDb.findAll(DBObject.class, collection);
+        int sizeBefore = docs.size();
+
+        String json = "{\"name\":\"Item A\"}\n{\"name\":\"Item B\"}";
+        List<String> jsonLines = ImportUtils.lines(json);
+        service.importTo(collection, jsonLines);
+
+        docs = mongoDb.findAll(DBObject.class, collection);
+        int sizeAfter = docs.size();
+        assertThat(sizeAfter - sizeBefore).isEqualTo(jsonLines.size());
+    }
+
+    @Test
+    public void givenJsonFile_whenClasspathSource_thenDocumentImported() {
+        String collection = "movies";
+        List<DBObject> docs = mongoDb.findAll(DBObject.class, collection);
+        int sizeBefore = docs.size();
+
+        String resource = "boot.json.convertfile/movies.json.log";
+        List<String> jsonLines = ImportUtils.linesFromResource(resource);
+        service.importTo(collection, jsonLines);
+
+        docs = mongoDb.findAll(DBObject.class, collection);
+        int sizeAfter = docs.size();
+        assertThat(sizeAfter - sizeBefore).isEqualTo(jsonLines.size());
+    }
+
+    @Test
+    public void givenJsonClasspathFile_whenCorrectlyTyped_thenDocumentImported() {
+        List<Book> books = bookRepository.findAll();
+        int sizeBefore = books.size();
+
+        String resource = "boot.json.convertfile/books.json.log";
+        List<String> jsonLines = ImportUtils.linesFromResource(resource);
+        service.importTo(Book.class, jsonLines);
+
+        books = bookRepository.findAll();
+        int sizeAfter = books.size();
+        assertThat(sizeAfter - sizeBefore).isEqualTo(jsonLines.size());
+    }
+
+    @Test
+    public void givenIncorrectlyTypedJson_whenUsingTypes_thenDocumentNotImported() {
+        List<Book> books = bookRepository.findAll();
+        int sizeBefore = books.size();
+
+        String resource = "boot.json.convertfile/movies.json.log";
+        List<String> jsonLines = ImportUtils.linesFromResource(resource);
+        service.importTo(Book.class, jsonLines);
+
+        books = bookRepository.findAll();
+        int sizeAfter = books.size();
+        assertThat(sizeAfter - sizeBefore).isEqualTo(0);
+    }
+
+    @Test
+    public void whenInvalidJson_thenDocumentNotImported() {
+        String collection = "items";
+        List<DBObject> docs = mongoDb.findAll(DBObject.class, collection);
+        int sizeBefore = docs.size();
+
+        String json = "{name: Item A}\n{name: Item B}";
+        List<String> jsonLines = ImportUtils.lines(json);
+        service.importTo(collection, jsonLines);
+
+        docs = mongoDb.findAll(DBObject.class, collection);
+        int sizeAfter = docs.size();
+        assertThat(sizeAfter - sizeBefore).isEqualTo(0);
+    }
+}

persistence-modules/spring-boot-persistence-mongodb/src/test/resources/boot.json.convertfile/books.json.log

@@ -0,0 +1,2 @@
+{"name":"Movie A", "genre": "Comedy"}
+{"name":"Movie B", "genre": "Thriller"}

persistence-modules/spring-boot-persistence-mongodb/src/test/resources/boot.json.convertfile/movies.json.log

@@ -0,0 +1,3 @@
+{"title":"Movie A", "genre": "Comedy"}
+{"title":"Movie B", "genre": "Thriller"}
+{"_id": "mongo-id", "title":"Movie C", "genre": "Drama"}