data-service

src/main/java/com/sxtvs/open/api/news/controller/YoumeiDataController.java

@@ -11,6 +11,7 @@ import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
+import java.util.Arrays;
 import java.util.List;
 
 /**
@@ -38,4 +39,9 @@ public class YoumeiDataController {
     public void put2es(){
         youmeiDataService.putDataRun();
     }
+
+    @RequestMapping("category")
+    public List<String> category(){
+        return Arrays.asList("财经", "彩票", "房产", "股票", "教育", "科技", "社会", "时尚", "时政", "体育", "星座", "游戏", "娱乐");
+    }
 }

src/main/java/com/sxtvs/open/api/news/entity/CategoryData.java

@@ -0,0 +1,10 @@
+package com.sxtvs.open.api.news.entity;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+@EqualsAndHashCode(callSuper = true)
+@Data
+public class CategoryData extends YoumeiData{
+    private String category;
+}

src/main/java/com/sxtvs/open/api/news/entity/YoumeiEsData.java

@@ -3,25 +3,43 @@ package com.sxtvs.open.api.news.entity;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import jakarta.annotation.Resource;
 import lombok.Data;
-import lombok.EqualsAndHashCode;
 
 import java.util.HashMap;
 import java.util.Map;
 
-@EqualsAndHashCode(callSuper = true)
 @Data
-public class YoumeiEsData extends YoumeiData{
+public class YoumeiEsData{
 
+    private Long offset;
 
-    private Map<String, Object> raw;
+    private String objType;
+
+    private String contentId;
+
+    private String title;
+
+    private String publishTime;
+
+    private String content;
+
+    private String sourceWebsite;
+
+    private String sourceUrl;
+
+    private String author;
+
+    private String category;
+
+    private String summary;
+
+    private String url;
 
     public YoumeiEsData() {
 
     }
 
-    public static YoumeiEsData of(YoumeiData youmeiData){
+    public static YoumeiEsData of(CategoryData youmeiData){
         YoumeiEsData data = new YoumeiEsData();
         data.setAuthor(youmeiData.getAuthor());
         data.setContent(youmeiData.getContent());
@@ -31,11 +49,14 @@ public class YoumeiEsData extends YoumeiData{
         data.setPublishTime(youmeiData.getPublishTime());
         data.setSourceUrl(youmeiData.getSourceUrl());
         data.setSourceWebsite(youmeiData.getSourceWebsite());
-        data.setRaw(youmeiData.getRawData());
+        Map<String, Object> mp = getMapFromRaw(youmeiData.getRawData());
+        data.setSummary((String) mp.get("summary"));
+        data.setUrl((String) mp.get("url"));
+        data.setCategory(youmeiData.getCategory());
         return data;
     }
 
-    public void setRaw(String rawData) {
+    private static Map<String, Object> getMapFromRaw(String rawData) {
         Map<String, Object> map = new HashMap<>();
         try {
             ObjectMapper objectMapper = new ObjectMapper();
@@ -43,10 +64,7 @@ public class YoumeiEsData extends YoumeiData{
         } catch (JsonProcessingException e) {
             e.printStackTrace();
         }
-        this.raw = (Map<String, Object>) map.get("data");
+        return (Map<String, Object>) map.get("data");
     }
 
-    public Map<String, Object> getRaw() {
-        return this.raw;
-    }
 }

src/main/java/com/sxtvs/open/api/news/mapper/YoumeiDataMapper.java

@@ -1,8 +1,12 @@
 package com.sxtvs.open.api.news.mapper;
 
 import com.baomidou.dynamic.datasource.annotation.DS;
+import com.sxtvs.open.api.news.entity.CategoryData;
 import com.sxtvs.open.api.news.entity.YoumeiData;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
 
 /**
  * <p>
@@ -14,5 +18,5 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper;
  */
 @DS("news")
 public interface YoumeiDataMapper extends BaseMapper<YoumeiData> {
-
+    List<CategoryData> getCategoryList(@Param("offset") Long offset, @Param("limit") Integer limit);
 }

src/main/java/com/sxtvs/open/api/news/service/impl/YoumeiDataServiceImpl.java

@@ -15,6 +15,7 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.sxtvs.open.api.news.dto.DataRequestDTO;
+import com.sxtvs.open.api.news.entity.CategoryData;
 import com.sxtvs.open.api.news.entity.YoumeiData;
 import com.sxtvs.open.api.news.entity.YoumeiEsData;
 import com.sxtvs.open.api.news.entity.YoumeiOffset;
@@ -23,6 +24,8 @@ import com.sxtvs.open.api.news.service.IYoumeiDataService;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import jakarta.annotation.Resource;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.http.util.TextUtils;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 import java.io.IOException;
@@ -49,9 +52,13 @@ public class YoumeiDataServiceImpl extends ServiceImpl<YoumeiDataMapper, YoumeiD
     @Resource
     private YoumeiOffsetServiceImpl youmeiOffsetService;
 
+    @Resource
+    private ObjectMapper objectMapper;
+
 
     public Page<YoumeiEsData> search(DataRequestDTO dataRequestDTO){
-        SearchResponse<YoumeiData> response = null;
+
+        SearchResponse<YoumeiEsData> response = null;
         Page<YoumeiEsData> page = new Page<>(dataRequestDTO.getPage(), dataRequestDTO.getPageSize());
         int offset = (int) ((page.getCurrent() - 1) * page.getSize());
         DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
@@ -59,13 +66,16 @@ public class YoumeiDataServiceImpl extends ServiceImpl<YoumeiDataMapper, YoumeiD
         try {
             SearchRequest.Builder builder = new SearchRequest.Builder();
             builder.index("news_data").from(offset).size((int) page.getSize());
-            if(dataRequestDTO.getStart() != null){
 
+            // 日期
+            if(dataRequestDTO.getStart() != null){
                 String start = dataRequestDTO.getStart().format(dateTimeFormatter);
                 String end = dataRequestDTO.getEnd().format(dateTimeFormatter);
                 builder.query(Query.of(x->x.range(y -> y.field("publishTime").gte(JsonData.of(start)).lte(JsonData.of(end)))));
             }
-            if("".equals(dataRequestDTO.getKeywords())){
+
+            // 关键词
+            if(TextUtils.isEmpty(dataRequestDTO.getKeywords())){
                 builder.sort(SortOptions.of(
                         so -> so.field(fs->fs.field("offset").order(SortOrder.Desc))
                 ));
@@ -75,9 +85,16 @@ public class YoumeiDataServiceImpl extends ServiceImpl<YoumeiDataMapper, YoumeiD
                 ).minScore(10D);
             }
 
+            // 类型
+            if(!TextUtils.isEmpty(dataRequestDTO.getCategory())){
+                builder.query(
+                        Query.of(x -> x.term(y -> y.field("category").value(dataRequestDTO.getCategory())))
+                );
+            }
+
             SearchRequest searchRequest = builder.build();
 
-            response = elasticsearchClient.search(searchRequest, YoumeiData.class);
+            response = elasticsearchClient.search(searchRequest, YoumeiEsData.class);
         } catch (IOException e) {
             e.printStackTrace();
         }
@@ -86,7 +103,7 @@ public class YoumeiDataServiceImpl extends ServiceImpl<YoumeiDataMapper, YoumeiD
         if (response != null) {
             response.hits().hits().forEach(x -> {
                 assert x.source() != null;
-                data.add(YoumeiEsData.of(x.source()));
+                data.add(x.source());
             });
             if (response.hits().total() != null) {
                 total = response.hits().total().value();
@@ -108,7 +125,7 @@ public class YoumeiDataServiceImpl extends ServiceImpl<YoumeiDataMapper, YoumeiD
         long offset = youmeiOffset.getOffset() + 1L;
         long max = 0L;
         while (true){
-            List<YoumeiData> youmeiDataList = lambdaQuery().gt(YoumeiData::getOffset, offset).last("limit 1000").list();
+            List<CategoryData> youmeiDataList = baseMapper.getCategoryList(offset, 1000);
             log.info("本次数据{}条，offset从{}开始",youmeiDataList.size(), offset);
             if (youmeiDataList.size() == 0){
                 break;
@@ -117,7 +134,8 @@ public class YoumeiDataServiceImpl extends ServiceImpl<YoumeiDataMapper, YoumeiD
             List<BulkOperation> bulkOperations = new ArrayList<>();
 
             youmeiDataList.forEach(a -> {
-                bulkOperations.add(BulkOperation.of(b -> b.index(c -> c.id(String.valueOf(a.getOffset())).document(a))));
+                YoumeiEsData youmeiEsData = YoumeiEsData.of(a);
+                bulkOperations.add(BulkOperation.of(b -> b.index(c -> c.id(String.valueOf(youmeiEsData.getOffset())).document(youmeiEsData))));
             });
 
             try {

src/main/resources/com/sxtvs/YoumeiDataMapper.xml

@@ -1,5 +1,10 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
 <mapper namespace="com.sxtvs.open.api.news.mapper.YoumeiDataMapper">
-
+    <select id="getCategoryList" resultType="com.sxtvs.open.api.news.entity.CategoryData">
+        SELECT d.*, c.category FROM youmei_data d
+        LEFT JOIN youmei_data_category c ON d.`offset` = c.`offset`
+        WHERE d.offset >= #{offset}
+        LIMIT #{limit}
+    </select>
 </mapper>