data-service

src/main/java/com/sxtvs/open/api/news/controller/YoumeiDataController.java

@@ -3,6 +3,7 @@ package com.sxtvs.open.api.news.controller;
 import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import com.sxtvs.open.api.news.dto.DataRequestDTO;
 import com.sxtvs.open.api.news.entity.YoumeiData;
+import com.sxtvs.open.api.news.entity.YoumeiEsData;
 import com.sxtvs.open.api.news.service.impl.YoumeiDataServiceImpl;
 import com.sxtvs.open.core.auth.LoginRequired;
 import jakarta.annotation.Resource;
@@ -29,7 +30,7 @@ public class YoumeiDataController {
 
     @RequestMapping("search")
     @LoginRequired
-    public Page<YoumeiData> search(@RequestBody DataRequestDTO dataRequestDTO){
+    public Page<YoumeiEsData> search(@RequestBody DataRequestDTO dataRequestDTO){
         return youmeiDataService.search(dataRequestDTO);
     }
 }

src/main/java/com/sxtvs/open/api/news/dto/DataRequestDTO.java

@@ -8,7 +8,10 @@ import java.time.LocalDateTime;
 @Data
 public class DataRequestDTO {
     @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd HH:mm:ss")
-    private LocalDateTime dt;
+    private LocalDateTime start;
+
+    @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd HH:mm:ss")
+    private LocalDateTime end;
 
     private String category;
 

src/main/java/com/sxtvs/open/api/news/dto/HotRankDTO.java

@@ -7,7 +7,7 @@ import lombok.Setter;
 @Setter
 @Getter
 public class HotRankDTO {
-    private String rk;
+    private Integer rk;
 
     private String content;
 

src/main/java/com/sxtvs/open/api/news/entity/HotRank.java

@@ -30,7 +30,7 @@ public class HotRank implements Serializable {
 
     private String rankName;
 
-    private String rk;
+    private Integer rk;
 
     private String content;
 

src/main/java/com/sxtvs/open/api/news/entity/YoumeiData.java

@@ -52,20 +52,4 @@ public class YoumeiData {
     @JsonIgnore
     private LocalDateTime createTime;
 
-    @Override
-    public String toString() {
-        return "YoumeiData{" +
-                "offset=" + offset +
-                ", objType='" + objType + '\'' +
-                ", contentId='" + contentId + '\'' +
-                ", title='" + title + '\'' +
-                ", publishTime='" + publishTime + '\'' +
-                ", content='" + content + '\'' +
-                ", sourceWebsite='" + sourceWebsite + '\'' +
-                ", sourceUrl='" + sourceUrl + '\'' +
-                ", author='" + author + '\'' +
-                ", rawData='" + rawData + '\'' +
-                ", createTime=" + createTime +
-                '}';
-    }
 }

src/main/java/com/sxtvs/open/api/news/entity/YoumeiEsData.java

@@ -0,0 +1,52 @@
+package com.sxtvs.open.api.news.entity;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import jakarta.annotation.Resource;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@EqualsAndHashCode(callSuper = true)
+@Data
+public class YoumeiEsData extends YoumeiData{
+
+
+    private Map<String, Object> raw;
+
+    public YoumeiEsData() {
+
+    }
+
+    public static YoumeiEsData of(YoumeiData youmeiData){
+        YoumeiEsData data = new YoumeiEsData();
+        data.setAuthor(youmeiData.getAuthor());
+        data.setContent(youmeiData.getContent());
+        data.setContentId(youmeiData.getContentId());
+        data.setObjType(youmeiData.getObjType());
+        data.setOffset(youmeiData.getOffset());
+        data.setPublishTime(youmeiData.getPublishTime());
+        data.setSourceUrl(youmeiData.getSourceUrl());
+        data.setSourceWebsite(youmeiData.getSourceWebsite());
+        data.setRaw(youmeiData.getRawData());
+        return data;
+    }
+
+    public void setRaw(String rawData) {
+        Map<String, Object> map = new HashMap<>();
+        try {
+            ObjectMapper objectMapper = new ObjectMapper();
+            map = objectMapper.readValue(rawData, new TypeReference<>() {});
+        } catch (JsonProcessingException e) {
+            e.printStackTrace();
+        }
+        this.raw = (Map<String, Object>) map.get("data");
+    }
+
+    public Map<String, Object> getRaw() {
+        return this.raw;
+    }
+}

src/main/java/com/sxtvs/open/api/news/service/impl/HotRankServiceImpl.java

@@ -25,7 +25,7 @@ public class HotRankServiceImpl extends ServiceImpl<HotRankMapper, HotRank> impl
 
     public List<HotRankDTO> todayRank(){
         String dt = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyyMMdd"));
-        return lambdaQuery().eq(HotRank::getDt, dt).eq(HotRank::getRankName, "密度-头条热榜").orderByAsc(HotRank::getRk).list()
-                .stream().map(HotRankDTO::new).collect(Collectors.toList());
+        return lambdaQuery().eq(HotRank::getDt, dt).eq(HotRank::getRankName, "密度-头条热榜").orderByAsc(HotRank::getRk)
+                .last("limit 20").list().stream().map(HotRankDTO::new).collect(Collectors.toList());
     }
 }

src/main/java/com/sxtvs/open/api/news/service/impl/YoumeiDataServiceImpl.java

@@ -5,11 +5,16 @@ import co.elastic.clients.elasticsearch._types.SortOptions;
 import co.elastic.clients.elasticsearch._types.SortOrder;
 import co.elastic.clients.elasticsearch._types.query_dsl.MultiMatchQuery;
 import co.elastic.clients.elasticsearch._types.query_dsl.Query;
+import co.elastic.clients.elasticsearch._types.query_dsl.RangeQuery;
 import co.elastic.clients.elasticsearch.core.SearchRequest;
 import co.elastic.clients.elasticsearch.core.SearchResponse;
+import co.elastic.clients.json.JsonData;
 import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.sxtvs.open.api.news.dto.DataRequestDTO;
 import com.sxtvs.open.api.news.entity.YoumeiData;
+import com.sxtvs.open.api.news.entity.YoumeiEsData;
 import com.sxtvs.open.api.news.mapper.YoumeiDataMapper;
 import com.sxtvs.open.api.news.service.IYoumeiDataService;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
@@ -17,9 +22,11 @@ import jakarta.annotation.Resource;
 import org.springframework.stereotype.Service;
 
 import java.io.IOException;
+import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 
 /**
  * <p>
@@ -34,13 +41,24 @@ public class YoumeiDataServiceImpl extends ServiceImpl<YoumeiDataMapper, YoumeiD
     @Resource
     private ElasticsearchClient elasticsearchClient;
 
-    public Page<YoumeiData> search(DataRequestDTO dataRequestDTO){
+    @Resource
+    private ObjectMapper objectMapper;
+
+    public Page<YoumeiEsData> search(DataRequestDTO dataRequestDTO){
         SearchResponse<YoumeiData> response = null;
-        Page<YoumeiData> page = new Page<>(dataRequestDTO.getPage(), dataRequestDTO.getPageSize());
+        Page<YoumeiEsData> page = new Page<>(dataRequestDTO.getPage(), dataRequestDTO.getPageSize());
         int offset = (int) ((page.getCurrent() - 1) * page.getSize());
+        DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+
         try {
             SearchRequest.Builder builder = new SearchRequest.Builder();
             builder.index("news_data").from(offset).size((int) page.getSize());
+            if(dataRequestDTO.getStart() != null){
+
+                String start = dataRequestDTO.getStart().format(dateTimeFormatter);
+                String end = dataRequestDTO.getEnd().format(dateTimeFormatter);
+                builder.query(Query.of(x->x.range(y -> y.field("publishTime").gte(JsonData.of(start)).lte(JsonData.of(end)))));
+            }
             if("".equals(dataRequestDTO.getKeywords())){
                 builder.sort(SortOptions.of(
                         so -> so.field(fs->fs.field("offset").order(SortOrder.Desc))
@@ -56,10 +74,13 @@ public class YoumeiDataServiceImpl extends ServiceImpl<YoumeiDataMapper, YoumeiD
         } catch (IOException e) {
             e.printStackTrace();
         }
-        List<YoumeiData> data = new ArrayList<>();
+        List<YoumeiEsData> data = new ArrayList<>();
         long total = 0;
         if (response != null) {
-            response.hits().hits().forEach(x -> data.add(x.source()));
+            response.hits().hits().forEach(x -> {
+                assert x.source() != null;
+                data.add(YoumeiEsData.of(x.source()));
+            });
             if (response.hits().total() != null) {
                 total = response.hits().total().value();
             }

src/main/java/com/sxtvs/open/core/advice/APIResponseAdvice.java

@@ -61,7 +61,8 @@ public class APIResponseAdvice implements ResponseBodyAdvice<Object> {
     @ExceptionHandler(Throwable.class)
     @ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR)
     public APIResponse serverError(Throwable ex) {
-        logger.error("error", ex);
+//        logger.error("error", ex);
+        log.error(ex.getMessage(),ex);
         APIResponse apiResponse = new APIResponse();
         apiResponse.setCode(-1);
         apiResponse.setMessage("服务器运行异常");

src/main/java/com/sxtvs/open/job/PutEsJob.java

@@ -28,7 +28,7 @@ public class PutEsJob {
     @Resource
     private YoumeiOffsetServiceImpl youmeiOffsetService;
 
-    @Scheduled(cron = "0 21 * * * *")
+    @Scheduled(cron = "0 54 * * * *")
     public void putData()  {
         log.info("ES 入数据开始");
         YoumeiOffset youmeiOffset = youmeiOffsetService.getById(4);
@@ -42,7 +42,9 @@ public class PutEsJob {
             }
             List<BulkOperation> bulkOperations = new ArrayList<>();
 
-            youmeiDataList.forEach(a -> bulkOperations.add(BulkOperation.of(b -> b.index(c -> c.id(String.valueOf(a.getOffset())).document(a)))));
+            youmeiDataList.forEach(a -> {
+                bulkOperations.add(BulkOperation.of(b -> b.index(c -> c.id(String.valueOf(a.getOffset())).document(a))));
+            });
 
             try {
                 elasticsearchClient.bulk(BulkRequest.of(x ->x.index("news_data").operations(bulkOperations)));

src/test/data-service.http

@@ -8,19 +8,20 @@ Content-Type: application/json;charset=UTF-8
 }
 
 ### 热榜
-GET https://open.sxtvs.net/news/hotRank/list
-Authorization: d18b62bbb29b1d6146b1cb894f27d032
+GET http://localhost/news/hotRank/list
+Authorization: b6af95077bafeccb32ae7a65dadee47d
 
 ### 搜索
-POST https://open.sxtvs.net/news/data/search
+POST http://localhost/news/data/search
 Content-Type: application/json;charset=UTF-8
-Authorization: d18b62bbb29b1d6146b1cb894f27d032
+Authorization: b6af95077bafeccb32ae7a65dadee47d
 
 {
-"keywords": "拜登为何此时秘密访问乌克兰",
+
   "category": "分类",
   "city": "城市",
-  "dt": "2023-02-21 15:52:52",
+  "start": "2023-02-10 15:52:52",
+  "end": "2023-02-22 15:52:52",
   "page": 1,
   "pageSize": 10
 }

src/test/java/com/sxtvs/open/EsClientTests.java

@@ -39,7 +39,7 @@ public class EsClientTests {
     public void test1(){
         try {
             elasticsearchClient.indices().delete(c -> c.index("news_data"));
-            elasticsearchClient.indices().create(c -> c.index("news_data"));
+//            elasticsearchClient.indices().create(c -> c.index("news_data"));
             System.out.println(elasticsearchClient.indices().exists(b -> b.index("news_data")).value());
         } catch (IOException e) {
             e.printStackTrace();
@@ -69,7 +69,10 @@ public class EsClientTests {
 
         List<BulkOperation> bulkOperations = new ArrayList<>();
 
-        youmeiDataList.forEach(a -> bulkOperations.add(BulkOperation.of(b -> b.index(c -> c.id(String.valueOf(a.getOffset())).document(a)))));
+        youmeiDataList.forEach(a -> {
+
+            bulkOperations.add(BulkOperation.of(b -> b.index(c -> c.id(String.valueOf(a.getOffset())).document(a))));
+        });
 
         elasticsearchClient.bulk(BulkRequest.of(x ->x.index("news_data").operations(bulkOperations)));
     }