|
| 1 | += Spring Batch: Working With REST Resources |
| 2 | +:source-highlighter: highlight.js |
| 3 | +:toc: |
| 4 | +:nofooter: |
| 5 | +:icons: font |
| 6 | +:url-quickref: https://github.com/rashidi/spring-boot-tutorials/tree/master/batch-rest-repository |
| 7 | + |
| 8 | +Implement batch operation for REST resources with https://spring.io/projects/spring-batch[Spring Batch] |
| 9 | + |
| 10 | + |
| 11 | +== Background |
| 12 | +Spring Batch allows us to perform large volumes of records from several resources such as https://docs.spring.io/spring-batch/docs/current/api/org/springframework/batch/item/file/FlatFileItemReader.html[File], |
| 13 | +https://docs.spring.io/spring-batch/docs/current/api/org/springframework/batch/item/database/JpaPagingItemReader.html[Relational Database], and, |
| 14 | +https://docs.spring.io/spring-batch/docs/current/api/org/springframework/batch/item/json/JsonItemReader.html[JSON file] to name a few. |
| 15 | + |
| 16 | +In this article, we will explore how to implement batch operation that reads from REST resources with Spring Batch through `JsonItemReader`. We will retrieve a list of users from https://jsonplaceholder.typicode.com/users[JSON Placeholder] and save them into a database. |
| 17 | + |
| 18 | +== Job Configuration |
| 19 | +Next is to implement the job that will be responsible to read from REST resource and save them into a database. `Job` consists of `Step` and `Step` |
| 20 | +consists of `ItemReader` and `ItemWriter`. We will implement all of them in link:{url-quickref}/src/main/java/zin/rashidi/boot/batch/rest/user/UserJobConfiguration.java[UserJobConfiguration]. |
| 21 | + |
| 22 | +[source,java] |
| 23 | +---- |
| 24 | +@Configuration |
| 25 | +class UserJobConfiguration { |
| 26 | + private final JobRepository jobRepository; |
| 27 | + private final PlatformTransactionManager transactionManager; |
| 28 | + private final MongoOperations mongo; |
| 29 | + UserJobConfiguration(JobRepository jobRepository, PlatformTransactionManager transactionManager, MongoOperations mongo) { |
| 30 | + this.jobRepository = jobRepository; |
| 31 | + this.transactionManager = transactionManager; |
| 32 | + this.mongo = mongo; |
| 33 | + } |
| 34 | + @Bean |
| 35 | + public Job userJob() throws MalformedURLException { |
| 36 | + return new JobBuilder("userJob", jobRepository).start(step()).build(); |
| 37 | + } |
| 38 | + private Step step() throws MalformedURLException { |
| 39 | + return new StepBuilder("userStep", jobRepository) |
| 40 | + .<User, User>chunk(10, transactionManager) |
| 41 | + .reader(reader()) |
| 42 | + .writer(writer()) |
| 43 | + .build(); |
| 44 | + } |
| 45 | + private JsonItemReader<User> reader() throws MalformedURLException { |
| 46 | + JacksonJsonObjectReader<User> jsonObjectReader = new JacksonJsonObjectReader<>(User.class); |
| 47 | + jsonObjectReader.setMapper(new ObjectMapper()); |
| 48 | + return new JsonItemReaderBuilder<User>() |
| 49 | + .name("userReader") |
| 50 | + .jsonObjectReader(jsonObjectReader) |
| 51 | + .resource(new UrlResource("https://jsonplaceholder.typicode.com/users")) |
| 52 | + .build(); |
| 53 | + } |
| 54 | + private MongoItemWriter<User> writer() { |
| 55 | + return new MongoItemWriterBuilder<User>() |
| 56 | + .template(mongo) |
| 57 | + .build(); |
| 58 | + } |
| 59 | +} |
| 60 | +---- |
| 61 | + |
| 62 | +From the code above, we can see that a `URL` form of `Resource` is assigned to `JsonItemReader`. We will depend on `JacksonJsonObjectRader` to convert response from link:https://jsonplaceholder.typicode.com/users[JSON Placeholder] to `User` object. |
| 63 | + |
| 64 | +[source,java] |
| 65 | +---- |
| 66 | +@Configuration |
| 67 | +class UserJobConfiguration { |
| 68 | + private JsonItemReader<User> reader() throws MalformedURLException { |
| 69 | + JacksonJsonObjectReader<User> jsonObjectReader = new JacksonJsonObjectReader<>(User.class); |
| 70 | + jsonObjectReader.setMapper(new ObjectMapper()); |
| 71 | + return new JsonItemReaderBuilder<User>() |
| 72 | + .name("userReader") |
| 73 | + .jsonObjectReader(jsonObjectReader) |
| 74 | + .resource(new UrlResource("https://jsonplaceholder.typicode.com/users")) |
| 75 | + .build(); |
| 76 | + } |
| 77 | +} |
| 78 | +---- |
| 79 | + |
| 80 | +Now that we have implemented the `Job`, we can verify that it is working by executing an integration test. |
| 81 | + |
| 82 | +== Verification |
| 83 | +We will launch `userJob` which will retrieve list of `User` from https://jsonplaceholder.typicode.com/users[JSON Placeholder] and save them into a database. |
| 84 | +Once completed then we will verify that the database contains the expected number of users. |
| 85 | + |
| 86 | +[source,java] |
| 87 | +---- |
| 88 | +@Testcontainers |
| 89 | +@SpringBatchTest |
| 90 | +@SpringBootTest(classes = { BatchTestConfiguration.class, MongoTestConfiguration.class, UserJobConfiguration.class }, webEnvironment = NONE) |
| 91 | +class UserBatchJobTests { |
| 92 | + @Container |
| 93 | + @ServiceConnection |
| 94 | + private final static MySQLContainer<?> MYSQL_CONTAINER = new MySQLContainer<>("mysql:latest") |
| 95 | + .withInitScript("org/springframework/batch/core/schema-mysql.sql"); |
| 96 | + @Container |
| 97 | + @ServiceConnection |
| 98 | + private final static MongoDBContainer MONGO_DB_CONTAINER = new MongoDBContainer("mongo:latest"); |
| 99 | + @Autowired |
| 100 | + private JobLauncherTestUtils launcher; |
| 101 | + @Autowired |
| 102 | + private MongoOperations mongoOperations; |
| 103 | + @Test |
| 104 | + @DisplayName("Given there are 10 users returned from REST Service When the job is COMPLETED Then all users should be saved to MongoDB") |
| 105 | + void launch() { |
| 106 | + await().atMost(ofSeconds(30)).untilAsserted(() -> { |
| 107 | + var execution = launcher.launchJob(); |
| 108 | + assertThat(execution.getExitStatus()).isEqualTo(COMPLETED); |
| 109 | + }); |
| 110 | + var persistedUsers = mongoOperations.findAll(User.class); |
| 111 | + assertThat(persistedUsers).hasSize(10); |
| 112 | + } |
| 113 | +} |
| 114 | +---- |
| 115 | + |
| 116 | +Full implementation can be found in link:{url-quickref}/src/test/java/zin/rashidi/boot/batch/rest/user/UserBatchJobTests.java[UserBatchJobTests]. |
0 commit comments