Commit 148133b8 authored by zlt2000's avatar zlt2000

init

parents
.idea/
logs/
target/
*.iml
# Compiled class file
*.class
# Log file
*.log
# BlueJ files
*.ctxt
# Mobile Tools for Java (J2ME)
.mtj.tmp/
# Package Files #
*.jar
*.war
*.nar
*.ear
*.zip
*.tar.gz
*.rar
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
hs_err_pid*
\ No newline at end of file
# Spring Boot 集成 Elasticsearch 7.x + XPACK
开源的 `Elasticsearch` 是一个分布式的 `RESTful` 风格的搜索和数据分析引擎,是目前全文搜索引擎的首选。
网上关于集成 `ElasticSearch` 的教程大部都是讲使用 `TransportClient` 的,但是该客户端本身并不支持 `XPACK` 安全认证需要引入其他依赖扩展,同时在 `ElasticSearch7` 的版本中已被废弃,并且会在8.x版本中将被移除,官方建议使用:`High Level REST Client`
并且由于 `ElasticSearch7.x` 某些变动并不向下兼容旧版本,而最新版本的 `Spring Boot Starter` 所依赖的 `ElasticSearch` 客户端还是**6.x**的版本,所以集成的时候需要填不少的坑。
在本场 Chat 中,会包含以下内容:
* `Spring Boot` 集成 `High Level REST Client`
* `Spring Boot` 集成 `XPACK` 认证
* 通过解读源码,解决集成 `ElasticSearch 7.x` 时遇到的坑
* 自定义连接 `ElasticSearch` 的http连接池配置
* 使用 `Spring Data Elasticsearch` 管理索引
* 使用 `Spring Data Elasticsearch` 对索引数据的基本 CRUD 操作
* 使用 `Junit4` 编写所有方法的测试用例
* 以上内容的所有源码
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.zlt</groupId>
<artifactId>spring-boot-elasticsearch7</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<java.version>8</java.version>
<spring-boot-dependencies.version>2.2.6.RELEASE</spring-boot-dependencies.version>
<elasticsearch.version>7.6.2</elasticsearch.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-dependencies</artifactId>
<version>${spring-boot-dependencies.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>7.6.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
<version>7.6.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>7.6.2</version>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${spring-boot-dependencies.version}</version>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
<finalName>${project.artifactId}</finalName>
</build>
</project>
\ No newline at end of file
package org.zlt;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
/**
* @author zlt
* @date 2020/5/3
* <p>
* Blog: https://zlt2000.gitee.io
* Github: https://github.com/zlt2000
*/
@SpringBootApplication
public class ElasticsearchApp {
public static void main(String[] args) {
SpringApplication.run(ElasticsearchApp.class, args);
}
}
package org.zlt.elasticsearch.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.elasticsearch.core.EntityMapper;
import org.springframework.data.elasticsearch.core.ResultsMapper;
import org.springframework.data.elasticsearch.core.mapping.SimpleElasticsearchMappingContext;
/**
* @author zlt
* @date 2020/5/3
* <p>
* Blog: https://zlt2000.gitee.io
* Github: https://github.com/zlt2000
*/
@Configuration
public class ElasticsearchConifg {
@Bean
public ResultsMapper resultsMapper(SimpleElasticsearchMappingContext mappingContext, EntityMapper entityMapper) {
return new MyResultMapper(mappingContext, entityMapper);
}
}
package org.zlt.elasticsearch.config;
import com.fasterxml.jackson.core.JsonEncoding;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.get.MultiGetItemResponse;
import org.elasticsearch.action.get.MultiGetResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.document.DocumentField;
import org.elasticsearch.search.SearchHit;
import org.springframework.core.convert.ConversionService;
import org.springframework.core.convert.support.DefaultConversionService;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.ElasticsearchException;
import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.ScriptedField;
import org.springframework.data.elasticsearch.core.AbstractResultMapper;
import org.springframework.data.elasticsearch.core.DefaultEntityMapper;
import org.springframework.data.elasticsearch.core.EntityMapper;
import org.springframework.data.elasticsearch.core.aggregation.AggregatedPage;
import org.springframework.data.elasticsearch.core.aggregation.impl.AggregatedPageImpl;
import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentEntity;
import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentProperty;
import org.springframework.data.elasticsearch.core.mapping.SimpleElasticsearchMappingContext;
import org.springframework.data.mapping.PersistentPropertyAccessor;
import org.springframework.data.mapping.context.MappingContext;
import org.springframework.data.mapping.model.ConvertingPropertyAccessor;
import org.springframework.lang.Nullable;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
/**
* @author zlt
* @date 2020/5/3
* <p>
* Blog: https://zlt2000.gitee.io
* Github: https://github.com/zlt2000
*/
public class MyResultMapper extends AbstractResultMapper {
private final MappingContext<? extends ElasticsearchPersistentEntity<?>, ElasticsearchPersistentProperty> mappingContext;
private final ConversionService conversionService = new DefaultConversionService();
public MyResultMapper() {
this(new SimpleElasticsearchMappingContext());
}
public MyResultMapper(
MappingContext<? extends ElasticsearchPersistentEntity<?>, ElasticsearchPersistentProperty> mappingContext) {
this(mappingContext, initEntityMapper(mappingContext));
}
public MyResultMapper(EntityMapper entityMapper) {
this(new SimpleElasticsearchMappingContext(), entityMapper);
}
public MyResultMapper(
MappingContext<? extends ElasticsearchPersistentEntity<?>, ElasticsearchPersistentProperty> mappingContext,
@Nullable EntityMapper entityMapper) {
super(entityMapper != null ? entityMapper : initEntityMapper(mappingContext));
this.mappingContext = mappingContext;
}
private static EntityMapper initEntityMapper(
MappingContext<? extends ElasticsearchPersistentEntity<?>, ElasticsearchPersistentProperty> mappingContext) {
Assert.notNull(mappingContext, "MappingContext must not be null!");
return new DefaultEntityMapper(mappingContext);
}
@Override
public <T> AggregatedPage<T> mapResults(SearchResponse response, Class<T> clazz, Pageable pageable) {
long totalHits = response.getHits().getTotalHits().value;
float maxScore = response.getHits().getMaxScore();
List<T> results = new ArrayList<>();
for (SearchHit hit : response.getHits()) {
if (hit != null) {
T result = null;
String hitSourceAsString = hit.getSourceAsString();
if (!StringUtils.isEmpty(hitSourceAsString)) {
result = mapEntity(hitSourceAsString, clazz);
} else {
result = mapEntity(hit.getFields().values(), clazz);
}
setPersistentEntityId(result, hit.getId(), clazz);
setPersistentEntityVersion(result, hit.getVersion(), clazz);
setPersistentEntityScore(result, hit.getScore(), clazz);
populateScriptFields(result, hit);
results.add(result);
}
}
return new AggregatedPageImpl<T>(results, pageable, totalHits, response.getAggregations(), response.getScrollId(),
maxScore);
}
private <T> void populateScriptFields(T result, SearchHit hit) {
if (hit.getFields() != null && !hit.getFields().isEmpty() && result != null) {
for (java.lang.reflect.Field field : result.getClass().getDeclaredFields()) {
ScriptedField scriptedField = field.getAnnotation(ScriptedField.class);
if (scriptedField != null) {
String name = scriptedField.name().isEmpty() ? field.getName() : scriptedField.name();
DocumentField searchHitField = hit.getFields().get(name);
if (searchHitField != null) {
field.setAccessible(true);
try {
field.set(result, searchHitField.getValue());
} catch (IllegalArgumentException e) {
throw new ElasticsearchException(
"failed to set scripted field: " + name + " with value: " + searchHitField.getValue(), e);
} catch (IllegalAccessException e) {
throw new ElasticsearchException("failed to access scripted field: " + name, e);
}
}
}
}
}
}
private <T> T mapEntity(Collection<DocumentField> values, Class<T> clazz) {
return mapEntity(buildJSONFromFields(values), clazz);
}
private String buildJSONFromFields(Collection<DocumentField> values) {
JsonFactory nodeFactory = new JsonFactory();
try {
ByteArrayOutputStream stream = new ByteArrayOutputStream();
JsonGenerator generator = nodeFactory.createGenerator(stream, JsonEncoding.UTF8);
generator.writeStartObject();
for (DocumentField value : values) {
if (value.getValues().size() > 1) {
generator.writeArrayFieldStart(value.getName());
for (Object val : value.getValues()) {
generator.writeObject(val);
}
generator.writeEndArray();
} else {
generator.writeObjectField(value.getName(), value.getValue());
}
}
generator.writeEndObject();
generator.flush();
return new String(stream.toByteArray(), Charset.forName("UTF-8"));
} catch (IOException e) {
return null;
}
}
@Override
public <T> T mapResult(GetResponse response, Class<T> clazz) {
T result = mapEntity(response.getSourceAsString(), clazz);
if (result != null) {
setPersistentEntityId(result, response.getId(), clazz);
setPersistentEntityVersion(result, response.getVersion(), clazz);
}
return result;
}
@Override
public <T> List<T> mapResults(MultiGetResponse responses, Class<T> clazz) {
List<T> list = new ArrayList<>();
for (MultiGetItemResponse response : responses.getResponses()) {
if (!response.isFailed() && response.getResponse().isExists()) {
T result = mapEntity(response.getResponse().getSourceAsString(), clazz);
setPersistentEntityId(result, response.getResponse().getId(), clazz);
setPersistentEntityVersion(result, response.getResponse().getVersion(), clazz);
list.add(result);
}
}
return list;
}
private <T> void setPersistentEntityId(T result, String id, Class<T> clazz) {
if (clazz.isAnnotationPresent(Document.class)) {
ElasticsearchPersistentEntity<?> persistentEntity = mappingContext.getRequiredPersistentEntity(clazz);
ElasticsearchPersistentProperty idProperty = persistentEntity.getIdProperty();
PersistentPropertyAccessor<T> accessor = new ConvertingPropertyAccessor<>(
persistentEntity.getPropertyAccessor(result), conversionService);
// Only deal with String because ES generated Ids are strings !
if (idProperty != null && idProperty.getType().isAssignableFrom(String.class)) {
accessor.setProperty(idProperty, id);
}
}
}
private <T> void setPersistentEntityVersion(T result, long version, Class<T> clazz) {
if (clazz.isAnnotationPresent(Document.class)) {
ElasticsearchPersistentEntity<?> persistentEntity = mappingContext.getPersistentEntity(clazz);
ElasticsearchPersistentProperty versionProperty = persistentEntity.getVersionProperty();
// Only deal with Long because ES versions are longs !
if (versionProperty != null && versionProperty.getType().isAssignableFrom(Long.class)) {
// check that a version was actually returned in the response, -1 would indicate that
// a search didn't request the version ids in the response, which would be an issue
Assert.isTrue(version != -1, "Version in response is -1");
persistentEntity.getPropertyAccessor(result).setProperty(versionProperty, version);
}
}
}
private <T> void setPersistentEntityScore(T result, float score, Class<T> clazz) {
if (clazz.isAnnotationPresent(Document.class)) {
ElasticsearchPersistentEntity<?> entity = mappingContext.getRequiredPersistentEntity(clazz);
if (!entity.hasScoreProperty()) {
return;
}
entity.getPropertyAccessor(result) //
.setProperty(entity.getScoreProperty(), score);
}
}
}
package org.zlt.elasticsearch.config;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.client.RestClientBuilder;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.elasticsearch.rest.RestClientBuilderCustomizer;
import org.springframework.boot.autoconfigure.elasticsearch.rest.RestClientProperties;
import org.springframework.boot.context.properties.PropertyMapper;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* 自定义es连接池
*
* @author zlt
* @date 2020/5/3
* <p>
* Blog: https://zlt2000.gitee.io
* Github: https://github.com/zlt2000
*/
@Configuration
public class RestAutoConfigure {
/**
* 链接建立超时时间
*/
@Value("${zlt.connectTimeOut:1000}")
private Integer connectTimeOut;
/**
* 等待数据超时时间
*/
@Value("${zlt.socketTimeOut:30000}")
private Integer socketTimeOut;
/**
* 连接池获取连接的超时时间
*/
@Value("${zlt.connectionRequestTimeOut:500}")
private Integer connectionRequestTimeOut;
/**
* 最大连接数
*/
@Value("${zlt.maxConnectNum:30}")
private Integer maxConnectNum;
/**
* 最大路由连接数
*/
@Value("${zlt.maxConnectPerRoute:10}")
private Integer maxConnectPerRoute;
@Bean
public RestClientBuilderCustomizer restClientBuilderCustomizer(RestClientProperties restProperties) {
return (builder) -> {
setRequestConfig(builder);
setHttpClientConfig(builder, restProperties);
};
}
/**
* 异步httpclient连接延时配置
*/
private void setRequestConfig(RestClientBuilder builder) {
builder.setRequestConfigCallback(requestConfigBuilder -> {
requestConfigBuilder.setConnectTimeout(connectTimeOut)
.setSocketTimeout(socketTimeOut)
.setConnectionRequestTimeout(connectionRequestTimeOut);
return requestConfigBuilder;
});
}
/**
* 异步httpclient连接数配置
*/
private void setHttpClientConfig(RestClientBuilder builder, RestClientProperties restProperties){
builder.setHttpClientConfigCallback(httpClientBuilder -> {
httpClientBuilder.setMaxConnTotal(maxConnectNum)
.setMaxConnPerRoute(maxConnectPerRoute);
PropertyMapper map = PropertyMapper.get().alwaysApplyingWhenNonNull();
map.from(restProperties::getUsername).to(username -> {
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials(username, restProperties.getPassword()));
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
});
return httpClientBuilder;
});
}
}
package org.zlt.elasticsearch.model;
import lombok.Getter;
import lombok.Setter;
import org.springframework.data.annotation.Id;
import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldType;
import java.io.Serializable;
/**
* @author zlt
* @date 2020/5/3
* <p>
* Blog: https://zlt2000.gitee.io
* Github: https://github.com/zlt2000
*/
@Setter
@Getter
@Document(indexName = "my-user", type = "_doc", shards = 1, replicas = 0)
public class MyUser implements Serializable {
private static final long serialVersionUID = -1;
@Id
private Long id;
@Field(type = FieldType.Text)
private String username;
@Field(type = FieldType.Keyword)
private String sex;
@Field(type = FieldType.Integer)
private Integer age;
}
package org.zlt.elasticsearch.service;
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
import org.zlt.elasticsearch.model.MyUser;
/**
* @author zlt
* @date 2020/5/3
* <p>
* Blog: https://zlt2000.gitee.io
* Github: https://github.com/zlt2000
*/
public interface MyUserRepository extends ElasticsearchRepository<MyUser, String> {
}
package org.zlt.elasticsearch.service;
import org.elasticsearch.index.query.QueryBuilders;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
import org.springframework.data.elasticsearch.core.query.AliasQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.data.elasticsearch.core.query.SearchQuery;
import org.springframework.stereotype.Service;
import org.zlt.elasticsearch.model.MyUser;
import java.util.Optional;
/**
* @author zlt
* @date 2020/5/3
* <p>
* Blog: https://zlt2000.gitee.io
* Github: https://github.com/zlt2000
*/
@Service
public class MyUserService {
@Autowired
private ElasticsearchRestTemplate elasticsearchRestTemplate;
@Autowired
private MyUserRepository userRepository;
/**
* 创建/更新索引
*/
public boolean createOrUpdateIndex() {
boolean result = elasticsearchRestTemplate.createIndex(MyUser.class);
if (result) {
result = elasticsearchRestTemplate.putMapping(MyUser.class);
}
return result;
}
/**
* 删除索引
*/
public boolean deleteIndex() {
return elasticsearchRestTemplate.deleteIndex(MyUser.class);
}
/**
* 增加索引别名
*/
public boolean addAlias() {
AliasQuery query = new AliasQuery();
query.setIndexName("my-user");
query.setAliasName("user");
return elasticsearchRestTemplate.addAlias(query);
}
/**
* 新增或更新数据
*/
public void save(MyUser user) {
userRepository.save(user);
}
/**
* 删除数据
*/
public void deleteById(String id) {
userRepository.deleteById(id);
}
/**
* 通过id查找
*/
public Optional<MyUser> findById(String id) {
return userRepository.findById(id);
}
/**
* 分页查询
*/
public Page<MyUser> queryPage() {
SearchQuery query = new NativeSearchQueryBuilder()
.withQuery(QueryBuilders.matchQuery("username", "admin"))
.withPageable(PageRequest.of(0, 10))
.build();
return userRepository.search(query);
}
}
spring.application.name=spring-boot-elasticsearch7
spring.elasticsearch.rest.uris=192.168.28.130:9200
#ESûƾ֤(DZ)
spring.elasticsearch.rest.username=elastic
spring.elasticsearch.rest.password=Csbi1Hlixv6kpVMmY4IV
\ No newline at end of file
package org.zlt.elasticsearch.service;
import static org.assertj.core.api.Assertions.assertThat;
import org.junit.*;
import org.junit.runner.RunWith;
import org.junit.runners.MethodSorters;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.data.domain.Page;
import org.springframework.test.context.junit4.SpringRunner;
import org.zlt.elasticsearch.model.MyUser;
import java.util.Optional;
/**
* @author zlt
* @date 2020/5/3
* <p>
* Blog: https://zlt2000.gitee.io
* Github: https://github.com/zlt2000
*/
@RunWith(SpringRunner.class)
@SpringBootTest
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class MyUserServiceTest {
@Autowired
private MyUserService userService;
@Test
public void test1CreateIndex() {
boolean result = userService.createOrUpdateIndex();
System.out.println(result);
assertThat(result).isTrue();
}
@Test
public void test8DeleteIndex() {
boolean result = userService.deleteIndex();
System.out.println(result);
assertThat(result).isTrue();
}
@Test
public void test2AddAlias() {
boolean result = userService.addAlias();
System.out.println(result);
assertThat(result).isTrue();
}
@Test
public void test3InsertUser() {
MyUser user = new MyUser();
user.setId(1L);
user.setUsername("admin");
user.setSex("男");
user.setAge(25);
userService.save(user);
}
@Test
public void test4UpdateUser() {
MyUser user = new MyUser();
user.setId(1L);
user.setUsername("admin");
user.setSex("女");
user.setAge(20);
userService.save(user);
}
@Test
public void test5FindUserById() {
Optional<MyUser> userOpt = userService.findById("1");
assertThat(userOpt.isPresent()).isTrue();
}
@Test
public void test6QueryPage() {
Page<MyUser> page = userService.queryPage();
page.forEach(System.out::println);
assertThat(page.getTotalElements()).isGreaterThan(0);
}
@Test
public void test7DeleteUserById() {
userService.deleteById("1");
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment