yoho-search / yohomars-search-service · Commits

Go to a project

Authored by all4you
Commit 70412a1124acd120b165994a61fa5e1d4470f489 1 parent 16b642dc

update

Inline Side-by-side
Showing 10 changed files with 99 additions and 29 deletions
... ... @@ -2,16 +2,16 @@ package com.yohomars.search.index.builder.impls;
import com.yoho.search.dal.SocialUserMapper;
import com.yoho.search.dal.model.SocialUser;
import com.yoho.tools.common.redis.RedisValueHelper;
import com.yohomars.search.index.builder.IIndexBuilder;
import com.yohomars.search.utils.ISearchConstans;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Resource;
import java.util.*;
/**
* @author gris.wang
... ... @@ -23,19 +23,30 @@ public class SocialUserIndexBuilder extends IIndexBuilder {
@Autowired
private SocialUserMapper socialUserMapper;
@Autowired
private RedisValueHelper redisValueHelper;
private Long lastUpdateTime;
private Long getLastUpdateTime(){
return redisValueHelper.get(ISearchConstans.LastUpdateTimeRedisKey.socialUser.getKey(),Long.class);
}
@Override
public int getTotalCount() throws Exception {
return socialUserMapper.selectCount();
lastUpdateTime = getLastUpdateTime();
return socialUserMapper.selectCount(lastUpdateTime);
}
@Override
public List<?> getPageLists(int offset, int limit) throws Exception {
List<SocialUser> list = socialUserMapper.selectPageList(offset,limit);
List<Map<String, Object>> dataList = new ArrayList<Map<String, Object>>();
if (CollectionUtils.isNotEmpty(list)) {
for(SocialUser socialUser : list) {
dataList.add(beanToMap(socialUser));
}
List<SocialUser> list = socialUserMapper.selectPageList(offset,limit,lastUpdateTime);
if(CollectionUtils.isEmpty(list)){
return Collections.emptyList();
}
List<Map<String, Object>> dataList = new ArrayList<>(list.size());
for(SocialUser socialUser : list) {
dataList.add(beanToMap(socialUser));
}
return dataList;
}
... ...
... ... @@ -3,6 +3,7 @@ package com.yohomars.search.index.service.impl;
import com.yoho.search.dal.model.SearchParam;
import com.yoho.search.dal.model.SearchResult;
import com.yoho.tools.common.redis.RedisValueHelper;
import com.yohomars.search.es.IElasticsearchClient;
import com.yohomars.search.index.builder.IIndexBuilder;
import com.yohomars.search.index.config.ClientConfig;
... ... @@ -15,6 +16,8 @@ import com.yohomars.search.index.model.IYohoIndex;
import com.yohomars.search.index.model.impl.YohoIndexImpl;
import com.yohomars.search.index.service.IYohoIndexService;
import com.yohomars.search.utils.FileUtils;
import com.yohomars.search.utils.ISearchConstans;
import com.yohomars.search.utils.Index;
import com.yohomars.search.utils.JaxbBinder;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
... ... @@ -36,6 +39,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
@Component
... ... @@ -52,7 +56,8 @@ public class YohoIndexServiceImpl implements IYohoIndexService {
@Autowired
private YohoIndexDataLoader yohoIndexDataLoader;
@Autowired
private RedisValueHelper redisValueHelper;
// 索引配置文件
private String configFile = "index.xml";
... ... @@ -336,8 +341,7 @@ public class YohoIndexServiceImpl implements IYohoIndexService {
String yohoTemplateIndexName = yohoIndexHelper.genTempIndexName(yohoIndexName);
// 3、创建临时索引
INDEX_REBUILD_LOG.info("rebuild [{}],step=[3.createTempIndex begin],mappingContent=[{}],setting=[{}],properties=[{}]", yohoIndexName, yohoIndex.getMappingContent(),
yohoIndex.getProperties());
INDEX_REBUILD_LOG.info("rebuild [{}],step=[3.createTempIndex begin],mappingContent=[{}],setting=[{}],properties=[{}]", yohoIndexName, yohoIndex.getMappingContent(), yohoIndex.getProperties());
String tempIndexRealName = this.createIndex(yohoIndexName, yohoTemplateIndexName, false);
INDEX_REBUILD_LOG.info("rebuild [{}],step=[3.createTempIndex success],tempIndexRealName=[{}],", yohoIndexName, tempIndexRealName);
... ... @@ -375,6 +379,10 @@ public class YohoIndexServiceImpl implements IYohoIndexService {
} catch (Exception e) {
INDEX_REBUILD_LOG.error(e.getMessage(), e);
} finally {
if(Index.social_user.getIndexName().equals(yohoIndexName)){
String key = ISearchConstans.LastUpdateTimeRedisKey.socialUser.getKey();
redisValueHelper.set(key,System.currentTimeMillis(),1, TimeUnit.DAYS);
}
rebuildFlagService.updateBuilding(false);
}
}
... ...
... ... @@ -37,12 +37,6 @@ public class ISearchConstans {
public static final String ACTION_UPDATE = "0";
public static final String ACTION_DELETE = "1";
// INDEXS
/// public static final String INDEX_NAME_BIZAREA = "bizarea";
/// public static final String INDEX_NAME_COMMENT = "comment";
/// public static final String INDEX_NAME_LINE = "line";
/// public static final String INDEX_NAME_STORE = "store";
/// public static final String INDEX_NAME_TOPIC = "topic";
// TableNames
public static final String TABLE_NAME_BIZAREA = "bizarea";
... ... @@ -118,6 +112,19 @@ public class ISearchConstans {
idFieldMap.put(Index.social_user.getIndexName(), "id");
}
public enum LastUpdateTimeRedisKey{
socialUser("YH:MARS:SEARCH:SOCIAL_USER:LAST_UPDATE_TIME");
private String key;
LastUpdateTimeRedisKey(String key){
this.key = key;
}
public String getKey(){
return key;
}
}
public static String getKeyField(final String indexName) {
return idFieldMap.get(indexName);
}
... ... @@ -126,7 +133,6 @@ public class ISearchConstans {
return channel.substring(20).replace("_", "");
}
public static String getTableKeyField(final String tableName) {
return LISTENED_TABLE_NAMES.get(tableName);
}
... ...
... ... @@ -7,7 +7,7 @@ import java.util.List;
public interface SocialUserMapper {
List<SocialUser> selectPageList(@Param(value = "offset") Integer offset, @Param(value = "pageSize") Integer pageSize);
List<SocialUser> selectPageList(@Param(value = "offset") Integer offset, @Param(value = "pageSize") Integer pageSize,@Param(value = "updateTime") Long updateTime);
int selectCount();
int selectCount(@Param(value = "updateTime") Long updateTime);
}
\ No newline at end of file
... ...
... ... @@ -12,6 +12,9 @@
</sql>
<sql id="param">
where status=1 and is_virtual=0
<if test="updateTime!=null">
and update_time &gt; #{updateTime}
</if>
</sql>
<select id="selectPageList" resultMap="BaseResultMap" timeout="20000">
select
... ...
... ... @@ -15,6 +15,7 @@
<properties>
<project-name>yohomars-search</project-name>
<yohobuy-tools-version>0.0.4-SNAPSHOT</yohobuy-tools-version>
</properties>
<dependencies>
... ... @@ -32,6 +33,11 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.yoho.tools</groupId>
<artifactId>yohobuy-tools-common</artifactId>
<version>${yohobuy-tools-version}</version>
</dependency>
</dependencies>
<dependencyManagement>
... ...
... ... @@ -37,8 +37,9 @@ public class SearchService extends BaseService {
/**
* 搜索
*
* @param
* @param paramMap
* @param index
* @param isSuggest
* @return
* @throws Exception
*/
... ... @@ -112,8 +113,6 @@ public class SearchService extends BaseService {
return null;
}
/// logger.info("result={}", JSON.toJSONString(searchResult));
// 构造返回结果
Map<String, Object> dataMap = new LinkedHashMap<String, Object>();
dataMap.put("total", searchResult.getTotal());
... ...
... ... @@ -14,6 +14,8 @@
<context:component-scan base-package="com.yohomars.search.*" />
<context:component-scan base-package="com.yohomars.search.index.service.*" />
<context:component-scan base-package="com.yohomars.search.service.*" />
<context:component-scan base-package="com.yoho.tools.*" />
<context:component-scan base-package="com.yoho.core.*" />
<!-- 打开aop 注解 -->
<aop:aspectj-autoproxy proxy-target-class="true"/>
... ... @@ -26,4 +28,13 @@
</list>
</property>
</bean>
<bean id="requestFactory" class="org.springframework.http.client.SimpleClientHttpRequestFactory">
<property name="readTimeout" value="60000" />
<property name="connectTimeout" value="10000" />
</bean>
<bean id="restTemplate" class="org.springframework.web.client.RestTemplate">
<constructor-arg ref="requestFactory" />
</bean>
</beans>
\ No newline at end of file
... ...
... ... @@ -26,4 +26,17 @@ search.default.field.socialUser=nick_name^1000
web.context=yohomars-search
#zkAddress
zkAddress=192.168.102.216:2181
\ No newline at end of file
zkAddress=192.168.102.216:2181
# ******************** redis servers ********************
redis.proxy.address=192.168.102.17
redis.proxy.port=6380
redis.proxy.auth=
redis.readonly.proxy.address=192.168.102.17
redis.readonly.proxy.port=6380
redis.readonly.proxy.auth=
redis.notsync.twemproxy.addresses=192.168.102.17:6380,192.168.102.17:6380
redis.notsync.twemproxy.auth=
\ No newline at end of file
... ...
... ... @@ -26,4 +26,17 @@ search.default.field.socialUser=nick_name^1000
web.context=yohomars-search
#zkAddress
zkAddress=${zkAddress}
\ No newline at end of file
zkAddress=${zkAddress}
# ******************** redis servers ********************
redis.proxy.address=${redis.proxy.address}
redis.proxy.port=${redis.proxy.port}
redis.proxy.auth=${redis.proxy.auth}
redis.readonly.proxy.address=${redis.readonly.proxy.address}
redis.readonly.proxy.port=${redis.readonly.proxy.port}
redis.readonly.proxy.auth=${redis.readonly.proxy.auth}
redis.notsync.twemproxy.addresses=${redis.notsync.twemproxy.addresses}
redis.notsync.twemproxy.auth=${redis.notsync.twemproxy.auth}
\ No newline at end of file
... ...