iSharkFly-Docs
/
java-tutorials
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

BAEL-5403 - Import Data to MongoDB from Json File using Java 

* Project
* Integration tests
This commit is contained in:
Ulisses Lima 2022-03-11 20:11:54 -03:00
parent 3b043a30dc
commit 5b56b1ed39
11 changed files with 427 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/SpringBootPersistenceApplication.java
View File

@ -1,13 +1,59 @@
package com.baeldung;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.ApplicationContext;
import com.baeldung.boot.json.convertfile.ImportUtils;
import com.baeldung.boot.json.convertfile.service.ImportJsonService;
@SpringBootApplication
public class SpringBootPersistenceApplication {
public class SpringBootPersistenceApplication implements ApplicationRunner {
private Logger log = LogManager.getLogger(this.getClass());
private static final String RESOURCE_PREFIX = "classpath:";
@Autowired
private ApplicationContext context;
public static void main(String ... args) {
SpringApplication.run(SpringBootPersistenceApplication.class, args);
}
@Override
public void run(ApplicationArguments args) throws Exception {
if (args.containsOption("import")) {
if (!args.containsOption("collection"))
throw new IllegalArgumentException("required option: --collection with collection name when using --import");
String collection = args.getOptionValues("collection")
.get(0);
List<String> sources = args.getOptionValues("import");
for (String source : sources) {
List<String> jsonLines = new ArrayList<>();
if (source.startsWith(RESOURCE_PREFIX)) {
String resource = source.substring(RESOURCE_PREFIX.length());
jsonLines = ImportUtils.linesFromResource(resource);
} else {
jsonLines = ImportUtils.lines(new File(source));
}
if (jsonLines == null || jsonLines.isEmpty())
throw new IllegalArgumentException("no input to import");
ImportJsonService importService = context.getBean(ImportJsonService.class);
String result = importService.importTo(collection, jsonLines);
log.info(source + " - result: " + result);
}
}
}
}

60
persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/ImportUtils.java Normal file
View File

@ -0,0 +1,60 @@
package com.baeldung.boot.json.convertfile;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
import org.springframework.web.multipart.MultipartFile;
public class ImportUtils {
private static Logger log = LogManager.getLogger(ImportUtils.class);
public static List<String> lines(String json) {
if (json == null)
return Collections.emptyList();
String[] split = json.split("[\\r\\n]+");
return Arrays.asList(split);
}
public static List<String> lines(MultipartFile file) {
try {
Path tmp = Files.write(File.createTempFile("mongo-upload", null)
.toPath(), file.getBytes());
return Files.readAllLines(tmp);
} catch (IOException e) {
log.error("reading lines from " + file, e);
return null;
}
}
public static List<String> lines(File file) {
try {
return Files.readAllLines(file.toPath());
} catch (IOException e) {
log.error("reading lines from " + file, e);
return null;
}
}
public static List<String> linesFromResource(String resource) {
Resource input = new ClassPathResource(resource);
try {
Path path = input.getFile()
.toPath();
return Files.readAllLines(path);
} catch (IOException e) {
log.error("reading lines from classpath resource: " + resource, e);
return null;
}
}
}

9
persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/dao/BookRepository.java Normal file
View File

@ -0,0 +1,9 @@
package com.baeldung.boot.json.convertfile.dao;
import org.springframework.data.mongodb.repository.MongoRepository;
import com.baeldung.boot.json.convertfile.data.Book;
public interface BookRepository extends MongoRepository<Book, String> {
}

38
persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/data/Book.java Normal file
View File

@ -0,0 +1,38 @@
package com.baeldung.boot.json.convertfile.data;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
@Document("books")
public class Book {
@Id
private String id;
private String name;
private String genre;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getGenre() {
return genre;
}
public void setGenre(String genre) {
this.genre = genre;
}
}

74
persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/service/ImportJsonService.java Normal file
View File

@ -0,0 +1,74 @@
package com.baeldung.boot.json.convertfile.service;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bson.Document;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.dao.DataIntegrityViolationException;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.mongodb.MongoBulkWriteException;
@Service
public class ImportJsonService {
private Logger log = LogManager.getLogger(this.getClass());
@Autowired
private MongoTemplate mongo;
public String importTo(Class<?> type, List<String> jsonLines) {
List<Document> mongoDocs = generateMongoDocs(jsonLines, type);
String collection = type.getAnnotation(org.springframework.data.mongodb.core.mapping.Document.class)
.value();
int inserts = insertInto(collection, mongoDocs);
return inserts + "/" + jsonLines.size();
}
public String importTo(String collection, List<String> jsonLines) {
List<Document> mongoDocs = generateMongoDocs(jsonLines);
int inserts = insertInto(collection, mongoDocs);
return inserts + "/" + jsonLines.size();
}
private int insertInto(String collection, List<Document> mongoDocs) {
try {
Collection<Document> inserts = mongo.insert(mongoDocs, collection);
return inserts.size();
} catch (DataIntegrityViolationException e) {
log.error("importing docs", e);
if (e.getCause() instanceof MongoBulkWriteException) {
return ((MongoBulkWriteException) e.getCause()).getWriteResult()
.getInsertedCount();
}
return 0;
}
}
private List<Document> generateMongoDocs(List<String> lines) {
return generateMongoDocs(lines, null);
}
private <T> List<Document> generateMongoDocs(List<String> lines, Class<T> type) {
ObjectMapper mapper = new ObjectMapper();
List<Document> docs = new ArrayList<>();
for (String json : lines) {
try {
if (type != null) {
T v = mapper.readValue(json, type);
json = mapper.writeValueAsString(v);
}
docs.add(Document.parse(json));
} catch (Throwable e) {
log.error("parsing: " + json, e);
}
}
return docs;
}
}

51
persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/web/BookController.java Normal file
View File

@ -0,0 +1,51 @@
package com.baeldung.boot.json.convertfile.web;
import java.io.IOException;
import java.util.List;
import java.util.Optional;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestPart;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import com.baeldung.boot.json.convertfile.ImportUtils;
import com.baeldung.boot.json.convertfile.dao.BookRepository;
import com.baeldung.boot.json.convertfile.data.Book;
import com.baeldung.boot.json.convertfile.service.ImportJsonService;
@RestController
@RequestMapping("/books")
public class BookController {
@Autowired
private BookRepository books;
@Autowired
private ImportJsonService service;
@PostMapping
public Book postBook(@RequestBody Book book) throws IOException {
return books.insert(book);
}
@GetMapping
public List<Book> getBooks() {
return books.findAll();
}
@GetMapping("/{id}")
public Optional<Book> getBook(@PathVariable String id) {
return books.findById(id);
}
@PostMapping("/import/file")
public String postJsonFile(@RequestPart("parts") MultipartFile jsonStringsFile) throws IOException {
List<String> jsonLines = ImportUtils.lines(jsonStringsFile);
return service.importTo(Book.class, jsonLines);
}
}

35
persistence-modules/spring-boot-persistence-mongodb/src/main/java/com/baeldung/boot/json/convertfile/web/ImportJsonController.java Normal file
View File

@ -0,0 +1,35 @@
package com.baeldung.boot.json.convertfile.web;
import java.io.IOException;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestPart;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import com.baeldung.boot.json.convertfile.ImportUtils;
import com.baeldung.boot.json.convertfile.service.ImportJsonService;
@RestController
@RequestMapping("/import-json")
public class ImportJsonController {
@Autowired
private ImportJsonService service;
@PostMapping("/{collection}")
public String postJson(@RequestBody String jsonStrings, @PathVariable String collection) {
List<String> jsonLines = ImportUtils.lines(jsonStrings);
return service.importTo(collection, jsonLines);
}
@PostMapping("/file/{collection}")
public String postJsonFile(@RequestPart("parts") MultipartFile jsonStringsFile, @PathVariable String collection) throws IOException {
List<String> jsonLines = ImportUtils.lines(jsonStringsFile);
return service.importTo(collection, jsonLines);
}
}

3
persistence-modules/spring-boot-persistence-mongodb/src/main/resources/boot.json.convertfile/data.json.log Normal file
View File

@ -0,0 +1,3 @@
{"name":"Book A", "genre": "Comedy"}
{"name":"Book B", "genre": "Thriller"}
{"_id": "mongo-id", "name":"Book C", "genre": "Drama"}

105
persistence-modules/spring-boot-persistence-mongodb/src/test/java/com/baeldung/boot/json/convertfile/service/ImportJsonServiceIntegrationTest.java Normal file
View File

@ -0,0 +1,105 @@
package com.baeldung.boot.json.convertfile.service;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.List;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.junit4.SpringRunner;
import com.baeldung.boot.json.convertfile.ImportUtils;
import com.baeldung.boot.json.convertfile.dao.BookRepository;
import com.baeldung.boot.json.convertfile.data.Book;
import com.mongodb.DBObject;
@SpringBootTest
@DirtiesContext
@RunWith(SpringRunner.class)
public class ImportJsonServiceIntegrationTest {
@Autowired
private ImportJsonService service;
@Autowired
private MongoTemplate mongoDb;
@Autowired
BookRepository bookRepository;
@Test
public void givenJsonString_whenGenericType_thenDocumentImported() {
String collection = "items";
List<DBObject> docs = mongoDb.findAll(DBObject.class, collection);
int sizeBefore = docs.size();
String json = "{\"name\":\"Item A\"}\n{\"name\":\"Item B\"}";
List<String> jsonLines = ImportUtils.lines(json);
service.importTo(collection, jsonLines);
docs = mongoDb.findAll(DBObject.class, collection);
int sizeAfter = docs.size();
assertThat(sizeAfter - sizeBefore).isEqualTo(jsonLines.size());
}
@Test
public void givenJsonFile_whenClasspathSource_thenDocumentImported() {
String collection = "movies";
List<DBObject> docs = mongoDb.findAll(DBObject.class, collection);
int sizeBefore = docs.size();
String resource = "boot.json.convertfile/movies.json.log";
List<String> jsonLines = ImportUtils.linesFromResource(resource);
service.importTo(collection, jsonLines);
docs = mongoDb.findAll(DBObject.class, collection);
int sizeAfter = docs.size();
assertThat(sizeAfter - sizeBefore).isEqualTo(jsonLines.size());
}
@Test
public void givenJsonClasspathFile_whenCorrectlyTyped_thenDocumentImported() {
List<Book> books = bookRepository.findAll();
int sizeBefore = books.size();
String resource = "boot.json.convertfile/books.json.log";
List<String> jsonLines = ImportUtils.linesFromResource(resource);
service.importTo(Book.class, jsonLines);
books = bookRepository.findAll();
int sizeAfter = books.size();
assertThat(sizeAfter - sizeBefore).isEqualTo(jsonLines.size());
}
@Test
public void givenIncorrectlyTypedJson_whenUsingTypes_thenDocumentNotImported() {
List<Book> books = bookRepository.findAll();
int sizeBefore = books.size();
String resource = "boot.json.convertfile/movies.json.log";
List<String> jsonLines = ImportUtils.linesFromResource(resource);
service.importTo(Book.class, jsonLines);
books = bookRepository.findAll();
int sizeAfter = books.size();
assertThat(sizeAfter - sizeBefore).isEqualTo(0);
}
@Test
public void whenInvalidJson_thenDocumentNotImported() {
String collection = "items";
List<DBObject> docs = mongoDb.findAll(DBObject.class, collection);
int sizeBefore = docs.size();
String json = "{name: Item A}\n{name: Item B}";
List<String> jsonLines = ImportUtils.lines(json);
service.importTo(collection, jsonLines);
docs = mongoDb.findAll(DBObject.class, collection);
int sizeAfter = docs.size();
assertThat(sizeAfter - sizeBefore).isEqualTo(0);
}
}

2
persistence-modules/spring-boot-persistence-mongodb/src/test/resources/boot.json.convertfile/books.json.log Normal file
View File

@ -0,0 +1,2 @@
{"name":"Movie A", "genre": "Comedy"}
{"name":"Movie B", "genre": "Thriller"}

3
persistence-modules/spring-boot-persistence-mongodb/src/test/resources/boot.json.convertfile/movies.json.log Normal file
View File

@ -0,0 +1,3 @@
{"title":"Movie A", "genre": "Comedy"}
{"title":"Movie B", "genre": "Thriller"}
{"_id": "mongo-id", "title":"Movie C", "genre": "Drama"}