yoho-search / yoho-search-consumer · Commits

Go to a project

Authored by hugufei
Commit 261392147c76382dc8edfb79ccbeeb8a86867f59 1 parent 88ae8f6b

使用ip2IndexInfo实现简单版本的_cat/shards功能

Inline Side-by-side
Showing 3 changed files with 198 additions and 135 deletions
package com.yoho.search.consumer.common;
import java.util.List;
import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken;
import org.elasticsearch.action.bulk.BulkResponse;
import com.yoho.search.core.es.model.ESBluk;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
/**
* 索引管理接口: 管理所有的索引和索引客户端，以及对外提供索引操作接口
*/
public interface IYohoIndexService {
/**
* 检查某索引的健康状态
*
* @param indexName
* @return
*/
boolean checkHealth(String indexName);
/**
* 通过索引名获取索引信息
*
* @param yohoIndexName 索引名称
* @return
*/
IYohoIndex getIndex(String yohoIndexName);
/**
* 执行索引重建
*
* @param yohoIndexName
*/
void rebuild(final String yohoIndexName);
/**
* 添加索引数据
*
* @param yohoIndexName 索引名称
* @param data 索引数据
*/
void addIndexData(String yohoIndexName, String id, Object data) throws Exception;
/**
* 删除索引数据
*
* @param yohoIndexName
*/
void deleteIndexData(String yohoIndexName, String id) throws Exception;
/**
* 更新索引数据
*
* @param yohoIndexName
*/
void updateIndexData(final String yohoIndexName, final String id, final Object data) throws Exception;
/**
* 执行搜索
*
* @param yohoIndexName
* @param searchParam
* @return
*/
SearchResult search(final String yohoIndexName, SearchParam searchParam);
/**
* 执行搜索
*
* @param yohoIndexName
* @param searchParams
* @return
*/
List<SearchResult> multiSearch(final String yohoIndexName, List<SearchParam> searchParams);
/**
* 获取分词结果
*
* @param yohoIndexName
* @param text
* @return MultiGetResponse
*/
List<AnalyzeToken> getAnalyzeTokens(final String yohoIndexName, String text, String analyzer);
/**
* 批量删除或者更新插入索引
*
* @param esBluks
* @return
*/
BulkResponse bulk(List<ESBluk> esBluks);
}
package com.yoho.search.consumer.common;
import java.util.List;
import com.yoho.search.core.es.IElasticsearchClient;
import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken;
import org.elasticsearch.action.bulk.BulkResponse;
import com.yoho.search.core.es.model.ESBluk;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
/**
* 索引管理接口: 管理所有的索引和索引客户端，以及对外提供索引操作接口
*/
public interface IYohoIndexService {
/**
* 获取集群客户端
*
* @return
*/
IElasticsearchClient getElasticsearchClient(String indexName);
/**
* 检查某索引的健康状态
*
* @param indexName
* @return
*/
boolean checkHealth(String indexName);
/**
* 通过索引名获取索引信息
*
* @param yohoIndexName 索引名称
* @return
*/
IYohoIndex getIndex(String yohoIndexName);
/**
* 执行索引重建
*
* @param yohoIndexName
*/
void rebuild(final String yohoIndexName);
/**
* 添加索引数据
*
* @param yohoIndexName 索引名称
* @param data 索引数据
*/
void addIndexData(String yohoIndexName, String id, Object data) throws Exception;
/**
* 删除索引数据
*
* @param yohoIndexName
*/
void deleteIndexData(String yohoIndexName, String id) throws Exception;
/**
* 更新索引数据
*
* @param yohoIndexName
*/
void updateIndexData(final String yohoIndexName, final String id, final Object data) throws Exception;
/**
* 执行搜索
*
* @param yohoIndexName
* @param searchParam
* @return
*/
SearchResult search(final String yohoIndexName, SearchParam searchParam);
/**
* 执行搜索
*
* @param yohoIndexName
* @param searchParams
* @return
*/
List<SearchResult> multiSearch(final String yohoIndexName, List<SearchParam> searchParams);
/**
* 获取分词结果
*
* @param yohoIndexName
* @param text
* @return MultiGetResponse
*/
List<AnalyzeToken> getAnalyzeTokens(final String yohoIndexName, String text, String analyzer);
/**
* 批量删除或者更新插入索引
*
* @param esBluks
* @return
*/
BulkResponse bulk(List<ESBluk> esBluks);
}
... ...
... ... @@ -124,6 +124,15 @@ public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEvent
}
@Override
public IElasticsearchClient getElasticsearchClient(String yohoIndexName) {
IYohoIndex index = this.nameToIndexMap.get(yohoIndexName);
if (index == null) {
return null;
}
return index.getIndexClient();
}
@Override
public IYohoIndex getIndex(String yohoIndexName) {
return this.nameToIndexMap.get(yohoIndexName);
}
... ... @@ -454,6 +463,7 @@ public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEvent
return bulkResponse;
}
@Override
public boolean checkHealth(String indexName) {
IYohoIndex index = this.nameToIndexMap.get(indexName);
... ...
package com.yoho.search.consumer.restapi;
import com.alibaba.fastjson.JSONObject;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.consumer.common.IYohoIndexService;
import com.yoho.search.consumer.service.logicService.personal.PersonalVectorVersionManager;
import com.yoho.search.consumer.service.logicService.tbl.util.StringUtils;
import com.yoho.search.core.es.IElasticsearchClient;
import com.yoho.search.core.personalized.service.BidataServiceCaller;
import org.apache.commons.collections.MapUtils;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
import java.util.HashMap;
import java.util.Map;
import java.util.*;
@Controller
public class ToolsController {
@Autowired
private PersonalVectorVersionManager personalVectorVersionManager;
@Autowired
private BidataServiceCaller bidataServiceCaller;
@RequestMapping(value = "/vectorVersion")
@ResponseBody
public Map<String, Object> vectorVersion(){
Map<String, Object> results = new HashMap<String, Object>();
//大数据目前推荐的版本
String bigDataRecomDateStr = personalVectorVersionManager.getBigDataRecomDateStr();
results.put("bigDataRecomDateStr", bigDataRecomDateStr==null?"":bigDataRecomDateStr);
//zk中目前的版本
String currentVersionInZk = personalVectorVersionManager.getCurrentVersionInZk();
results.put("currentVersionInZk", currentVersionInZk);
//经过计算目前可以使用的版本
String currentVersion = personalVectorVersionManager.getCurrentVersion();
results.put("currentVersion", currentVersion);
return results;
}
@RequestMapping(value = "/bigdataServiceTest")
@ResponseBody
public Map<String, Object> bigdataServiceTets(Integer uid){
Map<String, Object> results = new HashMap<String, Object>();
//大数据目前推荐的版本
String bigDataRecomDateStr = bidataServiceCaller.getBigDataRecomDateStr();
results.put("bigDataRecomDateStr", bigDataRecomDateStr==null?"":bigDataRecomDateStr);
results.put("userFavoriteSizes",bidataServiceCaller.getUserFavoriteSizes(uid.toString()));
results.put("userGenderFeature",bidataServiceCaller.getUserGenderFeature(uid.toString()));
results.put("userFavoriteSizes",bidataServiceCaller.getUserFavoriteSizes(uid.toString()));
results.put("userVectorFeature",bidataServiceCaller.getUserVectorFeature(uid.toString(),bigDataRecomDateStr));
results.put("userPersionalFactor",bidataServiceCaller.queryUserPersionalFactor(uid,null,null));
return results;
}
@Autowired
private PersonalVectorVersionManager personalVectorVersionManager;
@Autowired
private BidataServiceCaller bidataServiceCaller;
@Autowired
private IYohoIndexService yohoIndexService;
@RequestMapping(value = "/vectorVersion")
@ResponseBody
public Map<String, Object> vectorVersion() {
Map<String, Object> results = new HashMap<String, Object>();
//大数据目前推荐的版本
String bigDataRecomDateStr = personalVectorVersionManager.getBigDataRecomDateStr();
results.put("bigDataRecomDateStr", bigDataRecomDateStr == null ? "" : bigDataRecomDateStr);
//zk中目前的版本
String currentVersionInZk = personalVectorVersionManager.getCurrentVersionInZk();
results.put("currentVersionInZk", currentVersionInZk);
//经过计算目前可以使用的版本
String currentVersion = personalVectorVersionManager.getCurrentVersion();
results.put("currentVersion", currentVersion);
return results;
}
@RequestMapping(value = "/bigdataServiceTest")
@ResponseBody
public Map<String, Object> bigdataServiceTets(Integer uid) {
Map<String, Object> results = new HashMap<String, Object>();
//大数据目前推荐的版本
String bigDataRecomDateStr = bidataServiceCaller.getBigDataRecomDateStr();
results.put("bigDataRecomDateStr", bigDataRecomDateStr == null ? "" : bigDataRecomDateStr);
results.put("userFavoriteSizes", bidataServiceCaller.getUserFavoriteSizes(uid.toString()));
results.put("userGenderFeature", bidataServiceCaller.getUserGenderFeature(uid.toString()));
results.put("userFavoriteSizes", bidataServiceCaller.getUserFavoriteSizes(uid.toString()));
results.put("userVectorFeature", bidataServiceCaller.getUserVectorFeature(uid.toString(), bigDataRecomDateStr));
results.put("userPersionalFactor", bidataServiceCaller.queryUserPersionalFactor(uid, null, null));
return results;
}
@RequestMapping(value = "/ip2IndexInfo")
@ResponseBody
public Map<String, Object> nodeAndShardInfo() {
IElasticsearchClient client = yohoIndexService.getElasticsearchClient(ISearchConstants.INDEX_NAME_PRODUCT_INDEX);
ClusterState clusterState = client.getClusterStateResponse().getState();
DiscoveryNodes discoveryNodes = clusterState.nodes();
Map<String, List<Map<String, Object>>> ip2IndexInfo = new HashMap<>();
Set<String> allIps = new HashSet<>();
Set<String> hasPrimaryIps = new HashSet<>();
for (ShardRouting shard : clusterState.routingTable().allShards()) {
String indexName = shard.getIndexName();
String hostAddress = discoveryNodes.get(shard.currentNodeId()).getHostAddress();
boolean isPrimary = shard.primary();
if(isPrimary){
hasPrimaryIps.add(hostAddress);
}
allIps.add(hostAddress);
List<Map<String, Object>> indexInfoList = ip2IndexInfo.computeIfAbsent(hostAddress,a->new ArrayList<>());
JSONObject indexInfo = new JSONObject();
indexInfo.put("index_name", indexName);
indexInfo.put("node_id", shard.currentNodeId());
indexInfo.put("is_primary", isPrimary);
indexInfo.put("host_address", hostAddress);
indexInfoList.add(indexInfo);
}
Map<String, Object> result = new HashMap<>();
result.put("all",ip2IndexInfo);
result.put("hasPrimaryIps", StringUtils.join(hasPrimaryIps,","));
allIps.removeAll(hasPrimaryIps);
result.put("noPrimaryIps", StringUtils.join(allIps,","));
return result;
}
}
... ...