data-service

src/main/java/com/sxtvs/open/api/news/service/impl/YoumeiDataServiceImpl.java

@@ -42,14 +42,15 @@ public class YoumeiDataServiceImpl extends ServiceImpl<YoumeiDataMapper, YoumeiD
     private ElasticsearchClient elasticsearchClient;
 
     @Resource
-    private ObjectMapper objectMapper;
+    private YoumeiOffsetServiceImpl youmeiOffsetService;
+
 
     public Page<YoumeiEsData> search(DataRequestDTO dataRequestDTO){
         SearchResponse<YoumeiData> response = null;
         Page<YoumeiEsData> page = new Page<>(dataRequestDTO.getPage(), dataRequestDTO.getPageSize());
         int offset = (int) ((page.getCurrent() - 1) * page.getSize());
         DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
-
+        Long maxId = youmeiOffsetService.getById(4).getOffset();
         try {
             SearchRequest.Builder builder = new SearchRequest.Builder();
             builder.index("news_data").from(offset).size((int) page.getSize());
@@ -68,6 +69,7 @@ public class YoumeiDataServiceImpl extends ServiceImpl<YoumeiDataMapper, YoumeiD
                         Query.of(y -> y.multiMatch(MultiMatchQuery.of(z -> z.fields(Arrays.asList("title", "content")).query(dataRequestDTO.getKeywords()))))
                 ).minScore(10D);
             }
+            builder.postFilter(Query.of(x->x.range(y->y.field("offset").gte(JsonData.of(maxId - 10000)))));
             SearchRequest searchRequest = builder.build();
 
             response = elasticsearchClient.search(searchRequest, YoumeiData.class);

src/main/java/com/sxtvs/open/job/PutEsJob.java

@@ -37,6 +37,7 @@ public class PutEsJob {
         while (true){
             List<YoumeiData> youmeiDataList = youmeiDataService.lambdaQuery().gt(YoumeiData::getOffset, offset).last("limit 1000").list();
             log.info("本次数据{}条，offset从{}开始",youmeiDataList.size(), offset);
+            max = youmeiDataList.get(youmeiDataList.size()-1).getOffset();
             if (youmeiDataList.size() == 0){
                 break;
             }
@@ -51,7 +52,7 @@ public class PutEsJob {
             } catch (IOException e) {
                 e.printStackTrace();
             }
-            max = youmeiDataList.get(youmeiDataList.size()-1).getOffset();
+
             offset += 1000;
         }
         youmeiOffset.setOffset(max);