Skip to content

Commit 449f1fd

Browse files
dragonpooludomikula
dragonpoo
authored and
ludomikula
committed
convert to time-series collection
cron task for archiving
1 parent 6fece96 commit 449f1fd

File tree

11 files changed

+146
-45
lines changed

11 files changed

+146
-45
lines changed

‎server/api-service/lowcoder-domain/src/main/java/org/lowcoder/domain/application/model/ApplicationHistorySnapshotTS.java

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
package org.lowcoder.domain.application.model;
2+
3+
import lombok.Getter;
4+
import lombok.NoArgsConstructor;
5+
import lombok.Setter;
6+
import lombok.ToString;
7+
import org.lowcoder.sdk.models.HasIdAndAuditing;
8+
import org.springframework.data.mongodb.core.mapping.Document;
9+
10+
import java.util.Map;
11+
12+
@ToString(callSuper = true)
13+
@Document
14+
@Getter
15+
@Setter
16+
@NoArgsConstructor
17+
public class ApplicationHistorySnapshotTS extends HasIdAndAuditing {
18+
19+
private String applicationId;
20+
private Map<String, Object> dsl;
21+
private Map<String, Object> context;
22+
23+
}

‎server/api-service/lowcoder-domain/src/main/java/org/lowcoder/domain/application/repository/ApplicationHistorySnapshotRepository.java

+3-4
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,17 @@
11
package org.lowcoder.domain.application.repository;
22

3-
import org.lowcoder.domain.application.model.ApplicationHistorySnapshot;
3+
import org.lowcoder.domain.application.model.ApplicationHistorySnapshotTS;
44
import org.springframework.data.domain.Pageable;
55
import org.springframework.data.mongodb.repository.Query;
66
import org.springframework.data.mongodb.repository.ReactiveMongoRepository;
77
import org.springframework.stereotype.Repository;
8-
98
import reactor.core.publisher.Flux;
109
import reactor.core.publisher.Mono;
1110

1211
import java.time.Instant;
1312

1413
@Repository
15-
public interface ApplicationHistorySnapshotRepository extends ReactiveMongoRepository<ApplicationHistorySnapshot, String> {
14+
public interface ApplicationHistorySnapshotRepository extends ReactiveMongoRepository<ApplicationHistorySnapshotTS, String> {
1615

1716
@Query(value = "{ 'applicationId': ?0, $and: [" +
1817
"{$or: [ { 'context.operations': { $elemMatch: { 'compName': ?1 } } }, { $expr: { $eq: [?1, null] } } ]}, " +
@@ -21,7 +20,7 @@ public interface ApplicationHistorySnapshotRepository extends ReactiveMongoRepos
2120
"{$or: [ { 'createdAt': { $lte: ?4} }, { $expr: { $eq: [?4, null] } } ] } " +
2221
"]}",
2322
fields = "{applicationId : 1, context: 1, createdBy : 1, createdAt : 1}")
24-
Flux<ApplicationHistorySnapshot> findAllByApplicationId(String applicationId, String compName, String theme, Instant createdAtFrom, Instant createdAtTo, Pageable pageable);
23+
Flux<ApplicationHistorySnapshotTS> findAllByApplicationId(String applicationId, String compName, String theme, Instant createdAtFrom, Instant createdAtTo, Pageable pageable);
2524

2625
Mono<Long> countByApplicationId(String applicationId);
2726
}

‎server/api-service/lowcoder-domain/src/main/java/org/lowcoder/domain/application/service/ApplicationHistorySnapshotService.java

+6-8
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,20 @@
11
package org.lowcoder.domain.application.service;
22

3+
import org.lowcoder.domain.application.model.ApplicationHistorySnapshotTS;
4+
import org.springframework.data.domain.PageRequest;
5+
import reactor.core.publisher.Mono;
6+
37
import java.time.Instant;
48
import java.util.List;
59
import java.util.Map;
610

7-
import org.lowcoder.domain.application.model.ApplicationHistorySnapshot;
8-
import org.springframework.data.domain.PageRequest;
9-
10-
import org.springframework.web.bind.annotation.RequestParam;
11-
import reactor.core.publisher.Mono;
12-
1311
public interface ApplicationHistorySnapshotService {
1412

1513
Mono<Boolean> createHistorySnapshot(String applicationId, Map<String, Object> dsl, Map<String, Object> context, String userId);
1614

17-
Mono<List<ApplicationHistorySnapshot>> listAllHistorySnapshotBriefInfo(String applicationId, String compName, String theme, Instant from, Instant to, PageRequest pageRequest);
15+
Mono<List<ApplicationHistorySnapshotTS>> listAllHistorySnapshotBriefInfo(String applicationId, String compName, String theme, Instant from, Instant to, PageRequest pageRequest);
1816

1917
Mono<Long> countByApplicationId(String applicationId);
2018

21-
Mono<ApplicationHistorySnapshot> getHistorySnapshotDetail(String historySnapshotId);
19+
Mono<ApplicationHistorySnapshotTS> getHistorySnapshotDetail(String historySnapshotId);
2220
}

‎server/api-service/lowcoder-domain/src/main/java/org/lowcoder/domain/application/service/impl/ApplicationHistorySnapshotServiceImpl.java

+16-20
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,23 @@
11
package org.lowcoder.domain.application.service.impl;
22

3-
import static org.lowcoder.sdk.exception.BizError.INVALID_HISTORY_SNAPSHOT;
4-
import static org.lowcoder.sdk.util.ExceptionUtils.deferredError;
5-
import static org.lowcoder.sdk.util.ExceptionUtils.ofException;
6-
7-
import java.time.Instant;
8-
import java.util.List;
9-
import java.util.Map;
10-
113
import lombok.RequiredArgsConstructor;
12-
import org.lowcoder.domain.application.model.ApplicationHistorySnapshot;
4+
import org.lowcoder.domain.application.model.ApplicationHistorySnapshotTS;
135
import org.lowcoder.domain.application.repository.ApplicationHistorySnapshotRepository;
146
import org.lowcoder.domain.application.service.ApplicationHistorySnapshotService;
157
import org.lowcoder.sdk.exception.BizError;
16-
import org.springframework.beans.factory.annotation.Autowired;
17-
import org.springframework.context.annotation.Lazy;
188
import org.springframework.data.domain.PageRequest;
199
import org.springframework.data.domain.Sort.Direction;
2010
import org.springframework.stereotype.Service;
21-
22-
import org.springframework.web.bind.annotation.RequestParam;
2311
import reactor.core.publisher.Mono;
2412

13+
import java.time.Instant;
14+
import java.util.List;
15+
import java.util.Map;
16+
17+
import static org.lowcoder.sdk.exception.BizError.INVALID_HISTORY_SNAPSHOT;
18+
import static org.lowcoder.sdk.util.ExceptionUtils.deferredError;
19+
import static org.lowcoder.sdk.util.ExceptionUtils.ofException;
20+
2521
@RequiredArgsConstructor
2622
@Service
2723
public class ApplicationHistorySnapshotServiceImpl implements ApplicationHistorySnapshotService {
@@ -30,17 +26,17 @@ public class ApplicationHistorySnapshotServiceImpl implements ApplicationHistory
3026

3127
@Override
3228
public Mono<Boolean> createHistorySnapshot(String applicationId, Map<String, Object> dsl, Map<String, Object> context, String userId) {
33-
ApplicationHistorySnapshot applicationHistorySnapshot = new ApplicationHistorySnapshot();
34-
applicationHistorySnapshot.setApplicationId(applicationId);
35-
applicationHistorySnapshot.setDsl(dsl);
36-
applicationHistorySnapshot.setContext(context);
37-
return repository.save(applicationHistorySnapshot)
29+
ApplicationHistorySnapshotTS applicationHistorySnapshotTS = new ApplicationHistorySnapshotTS();
30+
applicationHistorySnapshotTS.setApplicationId(applicationId);
31+
applicationHistorySnapshotTS.setDsl(dsl);
32+
applicationHistorySnapshotTS.setContext(context);
33+
return repository.save(applicationHistorySnapshotTS)
3834
.thenReturn(true)
3935
.onErrorReturn(false);
4036
}
4137

4238
@Override
43-
public Mono<List<ApplicationHistorySnapshot>> listAllHistorySnapshotBriefInfo(String applicationId, String compName, String theme, Instant from, Instant to, PageRequest pageRequest) {
39+
public Mono<List<ApplicationHistorySnapshotTS>> listAllHistorySnapshotBriefInfo(String applicationId, String compName, String theme, Instant from, Instant to, PageRequest pageRequest) {
4440
return repository.findAllByApplicationId(applicationId, compName, theme, from, to, pageRequest.withSort(Direction.DESC, "id"))
4541
.collectList()
4642
.onErrorMap(Exception.class, e -> ofException(BizError.FETCH_HISTORY_SNAPSHOT_FAILURE, "FETCH_HISTORY_SNAPSHOT_FAILURE"));
@@ -55,7 +51,7 @@ public Mono<Long> countByApplicationId(String applicationId) {
5551

5652

5753
@Override
58-
public Mono<ApplicationHistorySnapshot> getHistorySnapshotDetail(String historySnapshotId) {
54+
public Mono<ApplicationHistorySnapshotTS> getHistorySnapshotDetail(String historySnapshotId) {
5955
return repository.findById(historySnapshotId)
6056
.switchIfEmpty(deferredError(INVALID_HISTORY_SNAPSHOT, "INVALID_HISTORY_SNAPSHOT", historySnapshotId));
6157
}

‎server/api-service/lowcoder-sdk/src/main/java/org/lowcoder/sdk/config/CommonConfig.java

+1
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@ public static class Marketplace {
160160
@Setter
161161
public static class Query {
162162
private long readStructureTimeout = 15000;
163+
private long appSnapshotKeepDuration = 30;
163164
}
164165

165166
@Data

‎server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/application/ApplicationHistorySnapshotController.java

+8-8
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
import org.lowcoder.api.home.SessionUserService;
1313
import org.lowcoder.api.util.Pagination;
1414
import org.lowcoder.domain.application.model.Application;
15-
import org.lowcoder.domain.application.model.ApplicationHistorySnapshot;
15+
import org.lowcoder.domain.application.model.ApplicationHistorySnapshotTS;
1616
import org.lowcoder.domain.application.service.ApplicationHistorySnapshotService;
1717
import org.lowcoder.domain.application.service.ApplicationService;
1818
import org.lowcoder.domain.permission.model.ResourceAction;
@@ -70,15 +70,15 @@ public Mono<ResponseView<Map<String, Object>>> listAllHistorySnapshotBriefInfo(@
7070
.flatMap(__ -> applicationHistorySnapshotService.listAllHistorySnapshotBriefInfo(applicationId, compName, theme, from, to, pagination.toPageRequest()))
7171
.flatMap(snapshotList -> {
7272
Mono<List<ApplicationHistorySnapshotBriefInfo>> snapshotBriefInfoList = multiBuild(snapshotList,
73-
ApplicationHistorySnapshot::getCreatedBy,
73+
ApplicationHistorySnapshotTS::getCreatedBy,
7474
userService::getByIds,
75-
(applicationHistorySnapshot, user) -> new ApplicationHistorySnapshotBriefInfo(
76-
applicationHistorySnapshot.getId(),
77-
applicationHistorySnapshot.getContext(),
78-
applicationHistorySnapshot.getCreatedBy(),
75+
(applicationHistorySnapshotTS, user) -> new ApplicationHistorySnapshotBriefInfo(
76+
applicationHistorySnapshotTS.getId(),
77+
applicationHistorySnapshotTS.getContext(),
78+
applicationHistorySnapshotTS.getCreatedBy(),
7979
user.getName(),
8080
user.getAvatarUrl(),
81-
applicationHistorySnapshot.getCreatedAt().toEpochMilli()
81+
applicationHistorySnapshotTS.getCreatedAt().toEpochMilli()
8282
)
8383
);
8484

@@ -97,7 +97,7 @@ public Mono<ResponseView<HistorySnapshotDslView>> getHistorySnapshotDsl(@PathVar
9797
.delayUntil(visitor -> resourcePermissionService.checkResourcePermissionWithError(visitor, applicationId,
9898
ResourceAction.EDIT_APPLICATIONS))
9999
.flatMap(__ -> applicationHistorySnapshotService.getHistorySnapshotDetail(snapshotId))
100-
.map(ApplicationHistorySnapshot::getDsl)
100+
.map(ApplicationHistorySnapshotTS::getDsl)
101101
.zipWhen(dsl -> applicationService.getAllDependentModulesFromDsl(dsl))
102102
.map(tuple -> {
103103
Map<String, Object> applicationDsl = tuple.getT1();

‎server/api-service/lowcoder-server/src/main/java/org/lowcoder/runner/migrations/DatabaseChangelog.java

+35
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
import lombok.extern.slf4j.Slf4j;
88
import org.bson.Document;
99
import org.lowcoder.domain.application.model.Application;
10+
import org.lowcoder.domain.application.model.ApplicationHistorySnapshot;
11+
import org.lowcoder.domain.application.model.ApplicationHistorySnapshotTS;
1012
import org.lowcoder.domain.bundle.model.Bundle;
1113
import org.lowcoder.domain.datasource.model.Datasource;
1214
import org.lowcoder.domain.datasource.model.DatasourceStructureDO;
@@ -27,9 +29,11 @@
2729
import org.lowcoder.runner.migrations.job.AddSuperAdminUser;
2830
import org.lowcoder.runner.migrations.job.CompleteAuthType;
2931
import org.lowcoder.runner.migrations.job.MigrateAuthConfigJob;
32+
import org.lowcoder.sdk.config.CommonConfig;
3033
import org.springframework.context.annotation.Profile;
3134
import org.springframework.data.domain.Sort;
3235
import org.springframework.data.mongodb.UncategorizedMongoDbException;
36+
import org.springframework.data.mongodb.core.CollectionOptions;
3337
import org.springframework.data.mongodb.core.DocumentCallbackHandler;
3438
import org.springframework.data.mongodb.core.index.CompoundIndexDefinition;
3539
import org.springframework.data.mongodb.core.index.Index;
@@ -39,6 +43,8 @@
3943
import org.springframework.data.mongodb.core.query.Update;
4044

4145
import java.time.Instant;
46+
import java.time.temporal.ChronoUnit;
47+
import java.util.List;
4248
import java.util.Set;
4349

4450
import static org.lowcoder.domain.util.QueryDslUtils.fieldName;
@@ -295,6 +301,35 @@ public void addGidIndexesUnique(MongockTemplate mongoTemplate) {
295301
ensureIndexes(mongoTemplate, LibraryQuery.class, makeIndex("gid").unique());
296302
}
297303

304+
@ChangeSet(order = "026", id = "add-time-series-snapshot-history", author = "")
305+
public void addTimeSeriesSnapshotHistory(MongockTemplate mongoTemplate, CommonConfig commonConfig) {
306+
// Create the time-series collection if it doesn't exist
307+
if (!mongoTemplate.collectionExists(ApplicationHistorySnapshotTS.class)) {
308+
mongoTemplate.createCollection(ApplicationHistorySnapshotTS.class, CollectionOptions.empty().timeSeries("createdAt"));
309+
}
310+
Instant thresholdDate = Instant.now().minus(commonConfig.getQuery().getAppSnapshotKeepDuration(), ChronoUnit.DAYS);
311+
List<ApplicationHistorySnapshot> snapshots = mongoTemplate.find(new Query().addCriteria(Criteria.where("createdAt").gte(thresholdDate)), ApplicationHistorySnapshot.class);
312+
snapshots.forEach(snapshot -> {
313+
ApplicationHistorySnapshotTS applicationHistorySnapshotTS = new ApplicationHistorySnapshotTS();
314+
applicationHistorySnapshotTS.setApplicationId(snapshot.getApplicationId());
315+
applicationHistorySnapshotTS.setDsl(snapshot.getDsl());
316+
applicationHistorySnapshotTS.setContext(snapshot.getContext());
317+
applicationHistorySnapshotTS.setCreatedAt(snapshot.getCreatedAt());
318+
applicationHistorySnapshotTS.setCreatedBy(snapshot.getCreatedBy());
319+
applicationHistorySnapshotTS.setModifiedBy(snapshot.getModifiedBy());
320+
applicationHistorySnapshotTS.setUpdatedAt(snapshot.getUpdatedAt());
321+
applicationHistorySnapshotTS.setId(snapshot.getId());
322+
mongoTemplate.insert(applicationHistorySnapshotTS);
323+
mongoTemplate.remove(snapshot);
324+
});
325+
326+
// Ensure indexes if needed
327+
ensureIndexes(mongoTemplate, ApplicationHistorySnapshotTS.class,
328+
makeIndex("applicationId"),
329+
makeIndex("createdAt")
330+
);
331+
}
332+
298333
private void addGidField(MongockTemplate mongoTemplate, String collectionName) {
299334
// Create a query to match all documents
300335
Query query = new Query();

‎server/api-service/lowcoder-server/src/main/java/org/lowcoder/runner/task/ArchiveSnapshotTask.java

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package org.lowcoder.runner.task;
2+
3+
import lombok.RequiredArgsConstructor;
4+
import lombok.extern.slf4j.Slf4j;
5+
import org.lowcoder.domain.application.model.ApplicationHistorySnapshot;
6+
import org.lowcoder.domain.application.model.ApplicationHistorySnapshotTS;
7+
import org.lowcoder.sdk.config.CommonConfig;
8+
import org.springframework.data.mongodb.core.MongoTemplate;
9+
import org.springframework.data.mongodb.core.query.Criteria;
10+
import org.springframework.data.mongodb.core.query.Query;
11+
import org.springframework.scheduling.annotation.Scheduled;
12+
import org.springframework.stereotype.Component;
13+
14+
import java.time.Instant;
15+
import java.time.temporal.ChronoUnit;
16+
import java.util.List;
17+
import java.util.concurrent.TimeUnit;
18+
19+
@Slf4j
20+
@RequiredArgsConstructor
21+
@Component
22+
public class ArchiveSnapshotTask {
23+
24+
private final CommonConfig commonConfig;
25+
private final MongoTemplate mongoTemplate;
26+
27+
@Scheduled(initialDelay = 1, fixedRate = 1, timeUnit = TimeUnit.DAYS)
28+
public void archive() {
29+
Instant thresholdDate = Instant.now().minus(commonConfig.getQuery().getAppSnapshotKeepDuration(), ChronoUnit.DAYS);
30+
List<ApplicationHistorySnapshotTS> snapshots = mongoTemplate.find(new Query().addCriteria(Criteria.where("createdAt").lte(thresholdDate)), ApplicationHistorySnapshotTS.class);
31+
snapshots.forEach(snapshot -> {
32+
ApplicationHistorySnapshot applicationHistorySnapshot = new ApplicationHistorySnapshot();
33+
applicationHistorySnapshot.setApplicationId(snapshot.getApplicationId());
34+
applicationHistorySnapshot.setDsl(snapshot.getDsl());
35+
applicationHistorySnapshot.setContext(snapshot.getContext());
36+
applicationHistorySnapshot.setCreatedAt(snapshot.getCreatedAt());
37+
applicationHistorySnapshot.setCreatedBy(snapshot.getCreatedBy());
38+
applicationHistorySnapshot.setModifiedBy(snapshot.getModifiedBy());
39+
applicationHistorySnapshot.setUpdatedAt(snapshot.getUpdatedAt());
40+
applicationHistorySnapshot.setId(snapshot.getId());
41+
mongoTemplate.insert(applicationHistorySnapshot);
42+
mongoTemplate.remove(snapshot);
43+
});
44+
}
45+
46+
}

‎server/api-service/lowcoder-server/src/main/resources/application-debug.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ common:
4747
notifications-email-sender: ${LOWCODER_EMAIL_NOTIFICATIONS_SENDER:info@localhost}
4848
cookie:
4949
max-age-in-hours: ${LOWCODER_COOKIE_MAX_AGE:24}
50+
query:
51+
app-snapshot-keep-duration: ${LOWCODER_QUERY_SNAPSHOT_ACTIVE_DURATION:30}
5052

5153
debug: true
5254

‎server/api-service/lowcoder-server/src/main/resources/application.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,8 @@ common:
8888
notifications-email-sender: ${LOWCODER_EMAIL_NOTIFICATIONS_SENDER:info@localhost}
8989
cookie:
9090
max-age-in-hours: ${LOWCODER_COOKIE_MAX_AGE:24}
91+
query:
92+
app-snapshot-keep-duration: ${LOWCODER_QUERY_SNAPSHOT_ACTIVE_DURATION:30}
9193

9294
material:
9395
mongodb-grid-fs:

‎server/api-service/lowcoder-server/src/test/java/org/lowcoder/api/service/impl/ApplicationHistorySnapshotServiceTest.java

+4-5
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,9 @@
22

33
import com.google.common.collect.ImmutableMap;
44
import lombok.extern.slf4j.Slf4j;
5-
import org.junit.Ignore;
65
import org.junit.Test;
76
import org.junit.runner.RunWith;
8-
import org.lowcoder.domain.application.model.ApplicationHistorySnapshot;
7+
import org.lowcoder.domain.application.model.ApplicationHistorySnapshotTS;
98
import org.lowcoder.domain.application.service.ApplicationHistorySnapshotService;
109
import org.lowcoder.sdk.models.HasIdAndAuditing;
1110
import org.springframework.beans.factory.annotation.Autowired;
@@ -48,8 +47,8 @@ public void testServiceMethods() {
4847
.assertNext(list -> {
4948
assertEquals(2, list.size());
5049

51-
ApplicationHistorySnapshot first = list.get(0);
52-
ApplicationHistorySnapshot second = list.get(1);
50+
ApplicationHistorySnapshotTS first = list.get(0);
51+
ApplicationHistorySnapshotTS second = list.get(1);
5352
assertTrue(first.getCreatedAt().isAfter(second.getCreatedAt()));
5453

5554
assertNull(first.getDsl());
@@ -67,7 +66,7 @@ public void testServiceMethods() {
6766
StepVerifier.create(service.listAllHistorySnapshotBriefInfo(applicationId, null, null, null, null, PageRequest.of(1, 1)))
6867
.assertNext(list -> {
6968
assertEquals(1, list.size());
70-
ApplicationHistorySnapshot one = list.get(0);
69+
ApplicationHistorySnapshotTS one = list.get(0);
7170
assertNull(one.getDsl());
7271
assertEquals(ImmutableMap.of("context", "context1"), one.getContext());
7372
assertEquals(applicationId, one.getApplicationId());

0 commit comments

Comments
 (0)