data-service

build.gradle

@@ -51,6 +51,13 @@ dependencies {
     compileOnly 'org.projectlombok:lombok'
     annotationProcessor 'org.projectlombok:lombok'
     testImplementation 'org.springframework.boot:spring-boot-starter-test'
+
+    implementation 'co.elastic.clients:elasticsearch-java:8.4.1'
+    implementation 'org.elasticsearch.client:elasticsearch-rest-client:8.4.1'
+    implementation 'com.fasterxml.jackson.core:jackson-databind:2.12.3'
+    implementation 'org.glassfish:jakarta.json:2.0.1'
+    implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.13.3'
+
 }
 
 tasks.named('test') {

src/main/java/com/sxtvs/open/OpenApplication.java

@@ -3,9 +3,11 @@ package com.sxtvs.open;
 import org.mybatis.spring.annotation.MapperScan;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.scheduling.annotation.EnableScheduling;
 
 @SpringBootApplication
 @MapperScan("com.sxtvs")
+@EnableScheduling
 public class OpenApplication {
 
     public static void main(String[] args) {

src/main/java/com/sxtvs/open/api/news/controller/HotRankController.java

@@ -0,0 +1,30 @@
+package com.sxtvs.open.api.news.controller;
+
+import com.sxtvs.open.api.news.entity.HotRank;
+import com.sxtvs.open.api.news.service.impl.HotRankServiceImpl;
+import jakarta.annotation.Resource;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.List;
+
+/**
+ * <p>
+ *  前端控制器
+ * </p>
+ *
+ * @author syj
+ * @since 2023-02-21
+ */
+@RestController
+@RequestMapping("/news/hotRank")
+public class HotRankController {
+
+    @Resource
+    private HotRankServiceImpl hotRankService;
+
+    @RequestMapping("list")
+    public List<HotRank> rank(){
+        return hotRankService.todayRank();
+    }
+}

src/main/java/com/sxtvs/open/api/news/controller/YoumeiDataController.java

@@ -0,0 +1,18 @@
+package com.sxtvs.open.api.news.controller;
+
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+/**
+ * <p>
+ *  前端控制器
+ * </p>
+ *
+ * @author syj
+ * @since 2023-02-20
+ */
+@RestController
+@RequestMapping("/news/youmeiData")
+public class YoumeiDataController {
+
+}

src/main/java/com/sxtvs/open/api/news/controller/YoumeiOffsetController.java

@@ -0,0 +1,18 @@
+package com.sxtvs.open.api.news.controller;
+
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+/**
+ * <p>
+ *  前端控制器
+ * </p>
+ *
+ * @author syj
+ * @since 2023-02-21
+ */
+@RestController
+@RequestMapping("/news/youmeiOffset")
+public class YoumeiOffsetController {
+
+}

src/main/java/com/sxtvs/open/api/news/entity/HotRank.java

@@ -0,0 +1,42 @@
+package com.sxtvs.open.api.news.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import java.io.Serializable;
+
+import lombok.Data;
+import lombok.Getter;
+import lombok.Setter;
+
+/**
+ * <p>
+ * 
+ * </p>
+ *
+ * @author syj
+ * @since 2023-02-21
+ */
+@Data
+@TableName("hot_rank")
+public class HotRank implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+
+    @TableId(value = "id", type = IdType.AUTO)
+    private Integer id;
+
+    private String platform;
+
+    private String rankName;
+
+    private String rk;
+
+    private String content;
+
+    private String href;
+
+    private String rsCount;
+
+    private String dt;
+}

src/main/java/com/sxtvs/open/api/news/entity/YoumeiData.java

@@ -0,0 +1,71 @@
+package com.sxtvs.open.api.news.entity;
+
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import java.io.Serializable;
+import java.time.LocalDateTime;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.data.annotation.Id;
+
+/**
+ * <p>
+ * 
+ * </p>
+ *
+ * @author syj
+ * @since 2023-02-20
+ */
+@Getter
+@Setter
+@TableName("youmei_data")
+@JsonIgnoreProperties(ignoreUnknown=true)
+public class YoumeiData {
+
+    @TableId
+    private Long offset;
+
+    private String objType;
+
+    private String contentId;
+
+    private String title;
+
+    private String publishTime;
+
+    private String content;
+
+    private String sourceWebsite;
+
+    private String sourceUrl;
+
+    private String author;
+
+    private String rawData;
+
+    @JsonIgnore
+    private LocalDateTime createTime;
+
+    @Override
+    public String toString() {
+        return "YoumeiData{" +
+                "offset=" + offset +
+                ", objType='" + objType + '\'' +
+                ", contentId='" + contentId + '\'' +
+                ", title='" + title + '\'' +
+                ", publishTime='" + publishTime + '\'' +
+                ", content='" + content + '\'' +
+                ", sourceWebsite='" + sourceWebsite + '\'' +
+                ", sourceUrl='" + sourceUrl + '\'' +
+                ", author='" + author + '\'' +
+                ", rawData='" + rawData + '\'' +
+                ", createTime=" + createTime +
+                '}';
+    }
+}

src/main/java/com/sxtvs/open/api/news/entity/YoumeiOffset.java

@@ -0,0 +1,28 @@
+package com.sxtvs.open.api.news.entity;
+
+import com.baomidou.mybatisplus.annotation.TableName;
+import java.io.Serializable;
+import lombok.Getter;
+import lombok.Setter;
+
+/**
+ * <p>
+ * 
+ * </p>
+ *
+ * @author syj
+ * @since 2023-02-21
+ */
+@Getter
+@Setter
+@TableName("youmei_offset")
+public class YoumeiOffset implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+
+    private Integer id;
+
+    private Long offset;
+
+    private String remark;
+}

src/main/java/com/sxtvs/open/api/news/mapper/HotRankMapper.java

@@ -0,0 +1,18 @@
+package com.sxtvs.open.api.news.mapper;
+
+import com.baomidou.dynamic.datasource.annotation.DS;
+import com.sxtvs.open.api.news.entity.HotRank;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+/**
+ * <p>
+ *  Mapper 接口
+ * </p>
+ *
+ * @author syj
+ * @since 2023-02-21
+ */
+@DS("collect")
+public interface HotRankMapper extends BaseMapper<HotRank> {
+
+}

src/main/java/com/sxtvs/open/api/news/mapper/YoumeiDataMapper.java

@@ -0,0 +1,18 @@
+package com.sxtvs.open.api.news.mapper;
+
+import com.baomidou.dynamic.datasource.annotation.DS;
+import com.sxtvs.open.api.news.entity.YoumeiData;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+/**
+ * <p>
+ *  Mapper 接口
+ * </p>
+ *
+ * @author syj
+ * @since 2023-02-20
+ */
+@DS("news")
+public interface YoumeiDataMapper extends BaseMapper<YoumeiData> {
+
+}

src/main/java/com/sxtvs/open/api/news/mapper/YoumeiOffsetMapper.java

@@ -0,0 +1,18 @@
+package com.sxtvs.open.api.news.mapper;
+
+import com.baomidou.dynamic.datasource.annotation.DS;
+import com.sxtvs.open.api.news.entity.YoumeiOffset;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+/**
+ * <p>
+ *  Mapper 接口
+ * </p>
+ *
+ * @author syj
+ * @since 2023-02-21
+ */
+@DS("news")
+public interface YoumeiOffsetMapper extends BaseMapper<YoumeiOffset> {
+
+}

src/main/java/com/sxtvs/open/api/news/service/IHotRankService.java

@@ -0,0 +1,16 @@
+package com.sxtvs.open.api.news.service;
+
+import com.sxtvs.open.api.news.entity.HotRank;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+/**
+ * <p>
+ *  服务类
+ * </p>
+ *
+ * @author syj
+ * @since 2023-02-21
+ */
+public interface IHotRankService extends IService<HotRank> {
+
+}

src/main/java/com/sxtvs/open/api/news/service/IYoumeiDataService.java

@@ -0,0 +1,16 @@
+package com.sxtvs.open.api.news.service;
+
+import com.sxtvs.open.api.news.entity.YoumeiData;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+/**
+ * <p>
+ *  服务类
+ * </p>
+ *
+ * @author syj
+ * @since 2023-02-20
+ */
+public interface IYoumeiDataService extends IService<YoumeiData> {
+
+}

src/main/java/com/sxtvs/open/api/news/service/IYoumeiOffsetService.java

@@ -0,0 +1,16 @@
+package com.sxtvs.open.api.news.service;
+
+import com.sxtvs.open.api.news.entity.YoumeiOffset;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+/**
+ * <p>
+ *  服务类
+ * </p>
+ *
+ * @author syj
+ * @since 2023-02-21
+ */
+public interface IYoumeiOffsetService extends IService<YoumeiOffset> {
+
+}

src/main/java/com/sxtvs/open/api/news/service/impl/HotRankServiceImpl.java

@@ -0,0 +1,28 @@
+package com.sxtvs.open.api.news.service.impl;
+
+import com.sxtvs.open.api.news.entity.HotRank;
+import com.sxtvs.open.api.news.mapper.HotRankMapper;
+import com.sxtvs.open.api.news.service.IHotRankService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+
+/**
+ * <p>
+ *  服务实现类
+ * </p>
+ *
+ * @author syj
+ * @since 2023-02-21
+ */
+@Service
+public class HotRankServiceImpl extends ServiceImpl<HotRankMapper, HotRank> implements IHotRankService {
+
+    public List<HotRank> todayRank(){
+        String dt = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyyMMdd"));
+        return lambdaQuery().eq(HotRank::getDt, dt).eq(HotRank::getRankName, "密度-头条热榜").orderByAsc(HotRank::getRk).list();
+    }
+}

src/main/java/com/sxtvs/open/api/news/service/impl/YoumeiDataServiceImpl.java

@@ -0,0 +1,20 @@
+package com.sxtvs.open.api.news.service.impl;
+
+import com.sxtvs.open.api.news.entity.YoumeiData;
+import com.sxtvs.open.api.news.mapper.YoumeiDataMapper;
+import com.sxtvs.open.api.news.service.IYoumeiDataService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+/**
+ * <p>
+ *  服务实现类
+ * </p>
+ *
+ * @author syj
+ * @since 2023-02-20
+ */
+@Service
+public class YoumeiDataServiceImpl extends ServiceImpl<YoumeiDataMapper, YoumeiData> implements IYoumeiDataService {
+
+}

src/main/java/com/sxtvs/open/api/news/service/impl/YoumeiOffsetServiceImpl.java

@@ -0,0 +1,20 @@
+package com.sxtvs.open.api.news.service.impl;
+
+import com.sxtvs.open.api.news.entity.YoumeiOffset;
+import com.sxtvs.open.api.news.mapper.YoumeiOffsetMapper;
+import com.sxtvs.open.api.news.service.IYoumeiOffsetService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+/**
+ * <p>
+ *  服务实现类
+ * </p>
+ *
+ * @author syj
+ * @since 2023-02-21
+ */
+@Service
+public class YoumeiOffsetServiceImpl extends ServiceImpl<YoumeiOffsetMapper, YoumeiOffset> implements IYoumeiOffsetService {
+
+}

src/main/java/com/sxtvs/open/core/conf/EsClientConfig.java

@@ -0,0 +1,153 @@
+package com.sxtvs.open.core.conf;
+
+import co.elastic.clients.elasticsearch.ElasticsearchClient;
+import co.elastic.clients.json.jackson.JacksonJsonpMapper;
+import co.elastic.clients.transport.ElasticsearchTransport;
+import co.elastic.clients.transport.rest_client.RestClientTransport;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.http.Header;
+import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.ssl.SSLContextBuilder;
+import org.apache.http.ssl.SSLContexts;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
+import org.elasticsearch.client.RestClientBuilder.HttpClientConfigCallback;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.util.StringUtils;
+
+import javax.net.ssl.SSLContext;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.KeyManagementException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
+import java.security.cert.CertificateFactory;
+
+@ConfigurationProperties(prefix = "elasticsearch") //配置的前缀
+@Configuration
+@Slf4j
+public class EsClientConfig {
+    @Setter
+    private String hosts;
+
+    @Setter
+    private String username;
+
+    @Setter
+    private String passwd;
+
+    @Setter
+    private String apikey;
+
+    /**
+     * 解析配置的字符串，转为HttpHost对象数组
+     * @return
+     */
+    private HttpHost[] toHttpHost() {
+        if (!StringUtils.hasLength(hosts)) {
+            throw new RuntimeException("invalid elasticsearch configuration");
+        }
+
+        String[] hostArray = hosts.split(",");
+        HttpHost[] httpHosts = new HttpHost[hostArray.length];
+        HttpHost httpHost;
+        for (int i = 0; i < hostArray.length; i++) {
+            String[] strings = hostArray[i].split(":");
+            httpHost = new HttpHost(strings[0], Integer.parseInt(strings[1]), "https");
+            httpHosts[i] = httpHost;
+        }
+
+        return httpHosts;
+    }
+
+    @Bean
+    public ElasticsearchClient clientByPasswd() throws Exception {
+        ElasticsearchTransport transport = getElasticsearchTransport(username, passwd, toHttpHost());
+        return new ElasticsearchClient(transport);
+    }
+
+    private static SSLContext buildSSLContext() {
+        ClassPathResource resource = new ClassPathResource("http_ca.crt");
+        SSLContext sslContext = null;
+        try {
+            CertificateFactory factory = CertificateFactory.getInstance("X.509");
+            Certificate trustedCa;
+            try (InputStream is = resource.getInputStream()) {
+                trustedCa = factory.generateCertificate(is);
+            }
+            KeyStore trustStore = KeyStore.getInstance("pkcs12");
+            trustStore.load(null, null);
+            trustStore.setCertificateEntry("ca", trustedCa);
+            SSLContextBuilder sslContextBuilder = SSLContexts.custom()
+                    .loadTrustMaterial(trustStore, null);
+            sslContext = sslContextBuilder.build();
+        } catch (CertificateException | IOException | KeyStoreException | NoSuchAlgorithmException |
+                KeyManagementException e) {
+            log.error("ES连接认证失败", e);
+        }
+
+        return sslContext;
+    }
+
+    private static ElasticsearchTransport getElasticsearchTransport(String username, String passwd, HttpHost...hosts) {
+        // 账号密码的配置
+        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+        credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, passwd));
+
+        // 自签证书的设置，并且还包含了账号密码
+        HttpClientConfigCallback callback = httpAsyncClientBuilder -> httpAsyncClientBuilder
+                .setSSLContext(buildSSLContext())
+                .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE)
+                .setDefaultCredentialsProvider(credentialsProvider);
+
+        // 用builder创建RestClient对象
+        RestClient client = RestClient
+                .builder(hosts)
+                .setHttpClientConfigCallback(callback)
+                .build();
+
+        return new RestClientTransport(client, new JacksonJsonpMapper());
+    }
+
+    /*private static ElasticsearchTransport getElasticsearchTransport(String apiKey, HttpHost...hosts) {
+        // 将ApiKey放入header中
+        Header[] headers = new Header[] {new BasicHeader("Authorization", "ApiKey " + apiKey)};
+
+        // es自签证书的设置
+        HttpClientConfigCallback callback = httpAsyncClientBuilder -> httpAsyncClientBuilder
+                .setSSLContext(buildSSLContext())
+                .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE);
+
+        // 用builder创建RestClient对象
+        RestClient client = RestClient
+                .builder(hosts)
+                .setHttpClientConfigCallback(callback)
+                .setDefaultHeaders(headers)
+                .build();
+
+        return new RestClientTransport(client, new JacksonJsonpMapper());
+    }
+
+//    @Bean
+    public ElasticsearchClient clientByApiKey() throws Exception {
+        ElasticsearchTransport transport = getElasticsearchTransport(apikey, toHttpHost());
+        return new ElasticsearchClient(transport);
+    }*/
+}

src/main/java/com/sxtvs/open/job/PutEsJob.java

@@ -0,0 +1,59 @@
+package com.sxtvs.open.job;
+
+import co.elastic.clients.elasticsearch.ElasticsearchClient;
+import co.elastic.clients.elasticsearch.core.BulkRequest;
+import co.elastic.clients.elasticsearch.core.bulk.BulkOperation;
+import com.sxtvs.open.api.news.entity.YoumeiData;
+import com.sxtvs.open.api.news.entity.YoumeiOffset;
+import com.sxtvs.open.api.news.service.impl.YoumeiDataServiceImpl;
+import com.sxtvs.open.api.news.service.impl.YoumeiOffsetServiceImpl;
+import jakarta.annotation.Resource;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+@Component
+@Slf4j
+public class PutEsJob {
+    @Resource
+    private ElasticsearchClient elasticsearchClient;
+
+    @Resource
+    private YoumeiDataServiceImpl youmeiDataService;
+
+    @Resource
+    private YoumeiOffsetServiceImpl youmeiOffsetService;
+
+    @Scheduled(cron = "0 21 * * * *")
+    public void putData()  {
+        log.info("ES 入数据开始");
+        YoumeiOffset youmeiOffset = youmeiOffsetService.getById(4);
+        long offset = youmeiOffset.getOffset() + 1L;
+        long max = 0L;
+        while (true){
+            List<YoumeiData> youmeiDataList = youmeiDataService.lambdaQuery().gt(YoumeiData::getOffset, offset).last("limit 1000").list();
+            log.info("本次数据{}条，offset从{}开始",youmeiDataList.size(), offset);
+            if (youmeiDataList.size() == 0){
+                break;
+            }
+            List<BulkOperation> bulkOperations = new ArrayList<>();
+
+            youmeiDataList.forEach(a -> bulkOperations.add(BulkOperation.of(b -> b.index(c -> c.id(String.valueOf(a.getOffset())).document(a)))));
+
+            try {
+                elasticsearchClient.bulk(BulkRequest.of(x ->x.index("news_data").operations(bulkOperations)));
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+            max = youmeiDataList.get(youmeiDataList.size()-1).getOffset();
+            offset += 1000;
+        }
+        youmeiOffset.setOffset(max);
+        youmeiOffsetService.updateById(youmeiOffset);
+        log.info("ES 入数据结束");
+    }
+}

src/main/resources/application.yml

@@ -39,6 +39,10 @@ spring:
           url: jdbc:mysql://rm-2vc3039h858t9ab7sfo.mysql.cn-chengdu.rds.aliyuncs.com:3306/collect?autoReconnect=true&useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=CONVERT_TO_NULL&useSSL=false&nullCatalogMeansCurrent=true
           username: cxzx
           password: sxtvs53$68HD
+        news:
+          url: jdbc:mysql://172.16.101.1:4000/youmei?autoReconnect=true&useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=CONVERT_TO_NULL&useSSL=false&nullCatalogMeansCurrent=true
+          username: root
+          password: msstar
   data:
     redis:
       host: r-2vch6mmrbgv8vjc44epd.redis.cn-chengdu.rds.aliyuncs.com
@@ -52,3 +56,9 @@ spring:
 
 server:
   port: 80
+
+elasticsearch:
+  hosts: 10.30.162.25:9200
+  username: elastic
+  passwd: Iqyb_f3RJ6V16IG3lS0t
+  apikey: RGM0S1hvWUI0aUtoa3RTR245SHo6cm9wZDBOUVBTaFM4UENHZE9BTUt3dw==

src/main/resources/com/sxtvs/HotRankMapper.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.sxtvs.open.api.news.mapper.HotRankMapper">
+
+</mapper>

src/main/resources/com/sxtvs/YoumeiDataMapper.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.sxtvs.open.api.news.mapper.YoumeiDataMapper">
+
+</mapper>

src/main/resources/com/sxtvs/YoumeiOffsetMapper.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.sxtvs.open.api.news.mapper.YoumeiOffsetMapper">
+
+</mapper>

src/main/resources/http_ca.crt

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFWTCCA0GgAwIBAgIUOgq8LqBdJtc2RqvQ6jKAmm16DMgwDQYJKoZIhvcNAQEL
+BQAwPDE6MDgGA1UEAxMxRWxhc3RpY3NlYXJjaCBzZWN1cml0eSBhdXRvLWNvbmZp
+Z3VyYXRpb24gSFRUUCBDQTAeFw0yMzAyMTcwMTQwMDhaFw0yNjAyMTYwMTQwMDha
+MDwxOjA4BgNVBAMTMUVsYXN0aWNzZWFyY2ggc2VjdXJpdHkgYXV0by1jb25maWd1
+cmF0aW9uIEhUVFAgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDS
+q3BorN+TaNevQbNV7vAymMBJdQEzXW3GHCQ3j9sxRGq70yFBgK14BuVqEr1Utuu0
+UoS7tDF9zye97DvVQDPdPqCiyrN6bZ+PPtu5CA9HV4HgDZh6ik/nNU0c5mSUktf5
+dvIf1O1zrQ7vMW43o2Jk9RJFPTfJv/OspCYlRFqK36g3Yyau1y+CFyr7IRnaxv8l
+oyMC9RYgjnPKPNViMM67GZ3fmFfKj+L+9Pk+USd2fOJzT8BN0tirFB7ZJZ1y74OO
+Kd+StZ2vn4J+wOXfTfk+/eXzBQFGDNDxCrynMCEQ1A4D6mwtSiBVcXF36g5VKzXC
+BrGN6gjkP4Iix43fO1e5FhDVWR1RqJ9gdjfzPsf1GddW3Rej4YCRSQfFR+uJ8odk
+qWkFqX5muX0MQ6xT0pGHxRR4JhczWN7xNC490qGproKl7+Hwh/24B11sgizD5AUz
+0tcUZgWtN+Q6c3/q+KBnpj2j8hskmtX71phTeoeNFCDtO4tMTGtCFVEPrhLfS2Z7
+lgbclmabRjE52N5fBpQSx55PUPcT3bVApBGCVQlEOsswZMU3VFWaBfcBdXL3/QYh
+mRRQs98lD9+CKGYfRmVs/Df1V1r0wHMaULyHgQPQKhgnewudO2vE2qPAMxTl32on
+aQk9Re/KGJuyf9l0cpU0G2qoiHyZhWwXCUoLsVP2rwIDAQABo1MwUTAdBgNVHQ4E
+FgQUBrMLDd9KdY9gxuBdauuhxB+u9NUwHwYDVR0jBBgwFoAUBrMLDd9KdY9gxuBd
+auuhxB+u9NUwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAgEAvXSk
++VEUyMGE9dhJH+9+SaP2iqIdZSdPrMAoeLIMWNmcMEmFixYhHXcQtoQHSksZkC/5
+Gm4tDcXCX1Q4veKNaIIkbBrve0UpUrBsPP9SGX+eU18A6xRmcuv8C5fh+WvKung2
+edcdJ6WjVxcTBztFhNAASL1qyprd6wu3h+wAxjK0p+7ZdthRepxtQ4slomIX4Oba
+kq3gUYjB2nZmmazDzECYsG0hSULH4eIUG5RpmBz1cFKjxTuWJVWlPcP6ireZ9B18
+UJiDI5ooZj7os36oelTtSXLaARfW/Q5WKOhprm4DRTw+U8NSwwrAd/KBLP1ZRqbU
+7+cabUaN179ICL1E3jilbeobySCWcxkpjQFzeG3OjNKi3tNL9YtO1gedhDNaks5a
+jEMQ+x1xbPOSfi3wAKYwnj8StvqHJHQnHRpGmgDG2MBMzauVpgNSORj1vUdbrWo7
+56+EKlz2An/qIQjE5otehXtrzzqoNIrQytUFIIKKK2bOSba3V6gOL14DOJiINByi
+gW118XKRta4czGOpZftq2QoaWcBK4UtxqixCtDiEkIqnYFGXF2s4oW3Fbbey5/16
+fhkO27/VQdlUMz5Z5BiWutoJfuigPS/cFNhgRF59EGWVSvOs5jcJCpqLcgA34Wqc
+JrINKjMQHl9SNE3whfNLGexZKiPB1MMTGGG9tEE=
+-----END CERTIFICATE-----

src/test/data-service.http

@@ -0,0 +1,2 @@
+### 热榜
+GET http://localhost/news/hotRank/list

src/test/java/com/sxtvs/open/EsClientTests.java

@@ -0,0 +1,90 @@
+package com.sxtvs.open;
+
+import co.elastic.clients.elasticsearch.ElasticsearchClient;
+import co.elastic.clients.elasticsearch._types.query_dsl.MatchQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.MultiMatchQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.Query;
+import co.elastic.clients.elasticsearch.core.*;
+import co.elastic.clients.elasticsearch.core.bulk.BulkOperation;
+import co.elastic.clients.elasticsearch.core.search.Hit;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.sxtvs.open.api.news.entity.YoumeiData;
+import com.sxtvs.open.api.news.service.impl.YoumeiDataServiceImpl;
+import jakarta.annotation.Resource;
+import org.junit.jupiter.api.Test;
+import org.springframework.boot.test.context.SpringBootTest;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+@SpringBootTest
+public class EsClientTests {
+
+    @Resource
+    private ElasticsearchClient elasticsearchClient;
+
+    @Resource
+    private YoumeiDataServiceImpl youmeiDataService;
+
+    @Resource
+    private ObjectMapper objectMapper;
+
+    @Test
+    public void test1(){
+        try {
+            elasticsearchClient.indices().delete(c -> c.index("news_data"));
+            elasticsearchClient.indices().create(c -> c.index("news_data"));
+            System.out.println(elasticsearchClient.indices().exists(b -> b.index("news_data")).value());
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    @Test
+    public void test2() throws IOException {
+        List<YoumeiData> youmeiDataList = youmeiDataService.lambdaQuery().gt(YoumeiData::getOffset, 7773600L).last("limit 20").list();
+
+        YoumeiData d = youmeiDataList.get(0);
+        String s = objectMapper.writeValueAsString(d);
+        System.out.println(s);
+        elasticsearchClient.index(IndexRequest.of(x->{
+            x.index("news_data");
+            x.id(String.valueOf(d.getOffset()));
+            x.document(d);
+            return x;
+        }));
+
+    }
+
+    @Test
+    public void insert() throws IOException, JsonProcessingException {
+
+        List<YoumeiData> youmeiDataList = youmeiDataService.lambdaQuery().gt(YoumeiData::getOffset, 7773600L).last("limit 20").list();
+
+        List<BulkOperation> bulkOperations = new ArrayList<>();
+
+        youmeiDataList.forEach(a -> bulkOperations.add(BulkOperation.of(b -> b.index(c -> c.id(String.valueOf(a.getOffset())).document(a)))));
+
+        elasticsearchClient.bulk(BulkRequest.of(x ->x.index("news_data").operations(bulkOperations)));
+    }
+
+    @Test
+    public void search() throws IOException {
+
+        SearchResponse<YoumeiData> response = elasticsearchClient.search(SearchRequest.of(x -> x.index("news_data").query(
+                Query.of(y -> y.multiMatch(MultiMatchQuery.of(z -> z.fields(Arrays.asList("title", "content")).query("拜登为何此时秘密访问乌克兰"))))
+        ).size(30)), YoumeiData.class);
+        List<YoumeiData> data = new ArrayList<>();
+        response.hits().hits().forEach(x -> {
+            System.out.println(x.score());
+            data.add(x.source());
+            System.out.println(x.source());
+        });
+
+        System.out.println(data.size());
+        System.out.println(data.stream().limit(2));
+    }
+}

src/test/java/com/sxtvs/open/GenCode.java

@@ -32,6 +32,54 @@ public class GenCode {
                 .execute();
     }
 
+    @Test
+    public void genCodeSyjPc() {
+        FastAutoGenerator.create("jdbc:mysql://172.16.101.1:4000/youmei",
+                "root", "msstar")
+                .globalConfig(builder -> {
+                    builder.author("syj") // 设置作者
+//                            .fileOverride() // 覆盖已生成文件
+                            .outputDir("D:\\dev\\java\\open\\src\\main\\java"); // 指定输出目录
+                })
+                .packageConfig(builder -> {
+                    builder.parent("com.sxtvs.open.api") // 设置父包名
+                            .moduleName("news") // 设置父包模块名
+                            .pathInfo(Collections.singletonMap(OutputFile.xml,
+                                    "D:\\dev\\java\\open\\src\\main\\resources\\com\\sxtvs")); // 设置mapperXml生成路径
+                })
+                .strategyConfig(builder -> {
+                    builder.addInclude("youmei_offset"); // 设置需要生成的表名
+                    builder.controllerBuilder().enableRestStyle();
+                    builder.entityBuilder().enableLombok();
+                })
+                .templateEngine(new FreemarkerTemplateEngine()) // 使用Freemarker引擎模板，默认的是Velocity引擎模板
+                .execute();
+    }
+
+    @Test
+    public void genCollectCodeSyjPc() {
+        FastAutoGenerator.create("jdbc:mysql://rm-2vc3039h858t9ab7sfo.mysql.cn-chengdu.rds.aliyuncs.com:3306/collect",
+                "cxzx", "sxtvs53$68HD")
+                .globalConfig(builder -> {
+                    builder.author("syj") // 设置作者
+//                            .fileOverride() // 覆盖已生成文件
+                            .outputDir("D:\\dev\\java\\open\\src\\main\\java"); // 指定输出目录
+                })
+                .packageConfig(builder -> {
+                    builder.parent("com.sxtvs.open.api") // 设置父包名
+                            .moduleName("news") // 设置父包模块名
+                            .pathInfo(Collections.singletonMap(OutputFile.xml,
+                                    "D:\\dev\\java\\open\\src\\main\\resources\\com\\sxtvs")); // 设置mapperXml生成路径
+                })
+                .strategyConfig(builder -> {
+                    builder.addInclude("hot_rank"); // 设置需要生成的表名
+                    builder.controllerBuilder().enableRestStyle();
+                    builder.entityBuilder().enableLombok();
+                })
+                .templateEngine(new FreemarkerTemplateEngine()) // 使用Freemarker引擎模板，默认的是Velocity引擎模板
+                .execute();
+    }
+
 //    @Test
 //    public void genCodeMac(){
 //        FastAutoGenerator.create("jdbc:mysql://rm-2vc3039h858t9ab7sfo.mysql.cn-chengdu.rds.aliyuncs.com:3306/collect",