Authored by all4you

add content

package com.yohomars.search.index.appender.impls;
import com.yohomars.search.index.appender.IndexAppender;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* @author gris.wang
* @since 2018/3/8
**/
@Component
public class ContentIndexAppender implements IndexAppender {
@Override
public Integer getMinId() throws Exception {
return null;
}
@Override
public Integer getMaxId() throws Exception {
return null;
}
@Override
public List<?> getListByLimit(int min, int limit) throws Exception {
return null;
}
@Override
public String getId(Object object) {
return null;
}
@Override
public String getRealId(Object object) {
return null;
}
}
... ...
... ... @@ -14,6 +14,7 @@ public interface IndexBuilder {
* @return 总数
* @throws Exception 异常
*/
@Deprecated
int getTotalCount() throws Exception;
/**
... ... @@ -23,6 +24,7 @@ public interface IndexBuilder {
* @return 数据列表
* @throws Exception 异常
*/
@Deprecated
List<?> getPageLists(int offset, int limit) throws Exception;
/**
... ...
package com.yohomars.search.index.builder.impls;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.yoho.search.dal.ContentMapper;
import com.yoho.search.dal.model.Content;
import com.yoho.search.dal.model.ContentTag;
import com.yohomars.search.index.builder.IndexBuilder;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* @author gris.wang
* @since 2018/3/8
**/
@Component
public class ContentIndexBuilder implements IndexBuilder{
@Autowired
private ContentMapper contentMapper;
@Override
public int getTotalCount() throws Exception {
return 0;
}
@Override
public List<?> getPageLists(int offset, int limit) throws Exception {
return null;
}
@Override
public Integer getMinId() throws Exception {
return contentMapper.selectMinId();
}
@Override
public Integer getMaxId() throws Exception {
return contentMapper.selectMaxId();
}
@Override
public List<?> getListByRange(int min, int max) throws Exception {
List<Content> list = contentMapper.selectListByRange(min,max);
List<Map<String, Object>> dataList = new ArrayList<Map<String, Object>>();
if(CollectionUtils.isNotEmpty(list)) {
List<Integer> contentIds = new ArrayList<>(list.size());
for (Content content : list) {
String title = content.getTitle();
String cover = content.getCover();
String summary = content.getSummary();
try {
if(StringUtils.isNotBlank(title) && title.startsWith("{")) {
JSONObject obj = JSON.parseObject(title);
title = obj.getString("s");
}
if(StringUtils.isNotBlank(cover) && cover.startsWith("{")) {
JSONObject obj = JSON.parseObject(cover);
cover = obj.getString("pic");
}
if(StringUtils.isNotBlank(summary) && summary.startsWith("{")) {
JSONObject obj = JSON.parseObject(summary);
summary = obj.getString("s");
}
}catch (Exception e){
// ignore
}
content.setTitle(title);
content.setCover(cover);
content.setSummary(summary);
contentIds.add(content.getCid());
}
if(CollectionUtils.isNotEmpty(contentIds)) {
List<ContentTag> contentTags = contentMapper.selectTagsByIds(contentIds);
if(CollectionUtils.isNotEmpty(contentTags)) {
for (Content content : list) {
for (ContentTag tag : contentTags) {
if(tag.getCid().equals(content.getId())){
content.addTagId(tag.getTagId());
content.addTagName(tag.getTagName());
content.increaseTagCount(1);
}
}
}
}
}
for (Content content : list) {
dataList.add(beanToMap(content));
}
}
return dataList;
}
@Override
public String getId(Object object) {
return ((Map)object).get("id").toString();
}
/**
* 拼装对象数据为map
* @param content
* @return
*/
private Map<String, Object> beanToMap(Content content) {
Map<String, Object> map = new LinkedHashMap<>();
map.put("id", content.getId());
map.put("cid", content.getCid());
map.put("type", content.getType());
map.put("app", content.getApp());
map.put("title",content.getTitle());
map.put("cover",content.getCover());
map.put("summary",content.getSummary());
map.put("relate_account",content.getRelateAccount());
map.put("tag_ids",content.getTagIds());
map.put("tag_names",content.getTagNames());
map.put("tag_count",content.getTagCount());
return map;
}
}
... ...
... ... @@ -5,6 +5,7 @@ import com.yoho.search.dal.StoreMapper;
import com.yoho.search.dal.model.Store;
import com.yohomars.search.index.builder.IndexBuilder;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
... ... @@ -58,9 +59,9 @@ public class StoreIndexBuilder implements IndexBuilder {
public List<?> getListByRange(int min, int max) throws Exception {
List<Store> list = storeMapper.selectListByRange(min, max);
List<Map<String, Object>> dataList = new ArrayList<Map<String, Object>>();
if (list != null && list.size() > 0) {
for (int i = 0; i < list.size(); i++) {
dataList.add(beanToMap(list.get(i)));
if(CollectionUtils.isNotEmpty(list)) {
for (Store store : list) {
dataList.add(beanToMap(store));
}
}
return dataList;
... ...
... ... @@ -32,7 +32,7 @@ public class RunnableQueueFactory {
public void addTask(String type, Runnable task){
BlockingQueue<Runnable> queue = queueMap.get(type);
if(queue==null){
queue = new ArrayBlockingQueue<>(MAX_QUEUE_CAPACITY);
queue = new LinkedBlockingQueue<>(MAX_QUEUE_CAPACITY);
putTask(queue,type,task);
queueMap.putIfAbsent(type,queue);
initQueue(type);
... ...
... ... @@ -128,6 +128,7 @@ public class ISearchConstans {
idFieldMap.put(Index.store.getIndexName(), "id");
idFieldMap.put(Index.topic.getIndexName(), "id");
idFieldMap.put(Index.social_user.getIndexName(), "id");
idFieldMap.put(Index.content.getIndexName(), "id");
}
public static String getKeyField(final String indexName) {
... ...
... ... @@ -28,7 +28,11 @@ public enum Index {
/**
* 资讯用户
*/
social_user("social_user","YH:MARS:SEARCH:LAST_UPDATE_TIME:SOCIAL_USER")
social_user("social_user","YH:MARS:SEARCH:LAST_UPDATE_TIME:SOCIAL_USER"),
/**
* 资讯文章
*/
content("content","YH:MARS:SEARCH:LAST_UPDATE_TIME:CONTENT")
;
... ...
package com.yoho.search.dal;
import com.yoho.search.dal.model.Content;
import com.yoho.search.dal.model.ContentTag;
import org.apache.ibatis.annotations.Param;
import java.util.List;
public interface ContentMapper {
int selectMinId();
int selectMaxId();
List<Content> selectListByRange(@Param(value = "minId") Integer minId, @Param(value = "maxId") Integer maxId);
List<ContentTag> selectTagsByIds(@Param(value = "contentIds") List<Integer> contentIds);
}
\ No newline at end of file
... ...
package com.yoho.search.dal.model;
import java.util.HashSet;
import java.util.Set;
/**
* @author gris.wang
* @since 2018/3/8
**/
public class Content {
private Integer id;
private Integer cid;
private Integer type;
private Integer app;
private String title;
private String cover;
private String summary;
private Integer relateAccount;
private Set<Integer> tagIds;
private Set<String> tagNames;
private Integer tagCount = 0;
public Integer getId() {
return id;
}
public void setId(Integer id) {
this.id = id;
}
public Integer getCid() {
return cid;
}
public void setCid(Integer cid) {
this.cid = cid;
}
public Integer getType() {
return type;
}
public void setType(Integer type) {
this.type = type;
}
public Integer getApp() {
return app;
}
public void setApp(Integer app) {
this.app = app;
}
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
public String getCover() {
return cover;
}
public void setCover(String cover) {
this.cover = cover;
}
public String getSummary() {
return summary;
}
public void setSummary(String summary) {
this.summary = summary;
}
public Integer getRelateAccount() {
return relateAccount;
}
public void setRelateAccount(Integer relateAccount) {
this.relateAccount = relateAccount;
}
public Set<Integer> getTagIds() {
return tagIds;
}
public void setTagIds(Set<Integer> tagIds) {
this.tagIds = tagIds;
}
public void addTagId(Integer tagId){
if(this.tagIds==null){
this.tagIds = new HashSet<>();
}
this.tagIds.add(tagId);
}
public Set<String> getTagNames() {
return tagNames;
}
public void setTagNames(Set<String> tagNames) {
this.tagNames = tagNames;
}
public void addTagName(String tagName){
if(this.tagNames==null){
this.tagNames = new HashSet<>();
}
this.tagNames.add(tagName);
}
public Integer getTagCount() {
return tagCount;
}
public void setTagCount(Integer tagCount) {
this.tagCount = tagCount;
}
public void increaseTagCount(Integer tagCount){
this.tagCount += tagCount;
}
}
... ...
package com.yoho.search.dal.model;
/**
* @author gris.wang
* @since 2018/3/8
**/
public class ContentTag{
private Integer cid;
private Integer tagId;
private String tagName;
public Integer getCid() {
return cid;
}
public void setCid(Integer cid) {
this.cid = cid;
}
public Integer getTagId() {
return tagId;
}
public void setTagId(Integer tagId) {
this.tagId = tagId;
}
public String getTagName() {
return tagName;
}
public void setTagName(String tagName) {
this.tagName = tagName;
}
}
\ No newline at end of file
... ...
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="com.yoho.search.dal.ContentMapper">
<resultMap id="BaseResultMap" type="com.yoho.search.dal.model.Content">
<result column="id" property="id" jdbcType="INTEGER"/>
<result column="cid" property="cid" jdbcType="INTEGER"/>
<result column="type" property="type" jdbcType="INTEGER"/>
<result column="app" property="app" jdbcType="INTEGER"/>
<result column="title" property="title" jdbcType="VARCHAR"/>
<result column="cover" property="cover" jdbcType="VARCHAR"/>
<result column="summary" property="summary" jdbcType="VARCHAR"/>
<result column="relate_account" property="relateAccount" jdbcType="INTEGER"/>
</resultMap>
<resultMap id="ContentTagsResultMap" type="com.yoho.search.dal.model.ContentTag">
<result column="cid" property="cid" jdbcType="INTEGER"/>
<result column="tag_id" property="tagId" jdbcType="INTEGER"/>
<result column="tag_name" property="tagName" jdbcType="VARCHAR"/>
</resultMap>
<sql id="Base_Column_List">
id,cid,`type`,app,title,cover,summary,relate_account
</sql>
<select id="selectMinId" resultType="java.lang.Integer" timeout="20000">
SELECT min(id) FROM tbl_merge_content_merge
</select>
<select id="selectMaxId" resultType="java.lang.Integer" timeout="20000">
SELECT max(id) FROM tbl_merge_content_merge
</select>
<select id="selectListByRange" resultMap="BaseResultMap" timeout="20000">
select
<include refid="Base_Column_List"/>
from tbl_merge_content_merge
where id BETWEEN #{minId} AND #{maxId}
</select>
<select id="selectTagsByIds" resultMap="ContentTagsResultMap" timeout="20000">
select ts.content_id as cid,t.id as tag_id,t.name as tag_name
from tbl_tags t,tbl_tags_show ts
where ts.tag_id=t.id and
ts.content_id
<foreach collection="contentIds" item="contentId" open="in (" separator="," close=")">
#{contentId}
</foreach>
union all
select ts.content_id as cid,t.id as tag_id,t.name as tag_name
from tbl_tags t,tbl_tags_show_girl ts
where ts.tag_id=t.id and
ts.content_id
<foreach collection="contentIds" item="contentId" open="in (" separator="," close=")">
#{contentId}
</foreach>
</select>
</mapper>
\ No newline at end of file
... ...
... ... @@ -25,21 +25,29 @@ public class IndexRebuildJob {
/**
* 定时任务重建所有索引(每1小时执行一次)
* 定时任务重建索引(每1小时执行一次)
* 该索引数据量大
*/
@Scheduled(cron = "0 0 */1 * * ?")
public void rebuildIndexJob() {
this.rebuildIndex(Index.social_user);
}
/**
* 定时任务增量更新所有索引(每10分钟执行一次)
* 定时任务重建索引(每10分钟执行一次)
* 该索引数据量小
*/
@Scheduled(cron = "0 */10 * * * ?")
public void appendIndexJob() {
// store的数据量少,也每10分钟直接重建一次
public void rebuildSmallIndexJob() {
this.rebuildIndex(Index.store);
this.rebuildIndex(Index.content);
}
/**
* 定时任务增量更新索引(每10分钟执行一次)
*/
@Scheduled(cron = "50 */10 * * * ?")
public void appendIndexJob() {
this.appendIndex(Index.social_user);
}
... ...
... ... @@ -17,11 +17,12 @@ datasources:
yh_cms:
servers:
- 172.16.4.138:3306
- 172.16.4.138:3306
- 192.168.102.17:3306
- 192.168.102.17:3306
username: yh_test
password: 9nm0icOwt6bMHjMusIfMLw==
daos:
- com.yoho.search.dal.SocialUserDetailMapper
- com.yoho.search.dal.ContentMapper
readOnlyInSlave: true
\ No newline at end of file
... ...
{
"content": {
"_all":{
"enabled":false
},
"_source":{
"enabled":true
},
"properties": {
"id": {
"type": "integer"
},
"cid": {
"type": "integer"
},
"type": {
"type": "integer"
},
"app": {
"type": "integer"
},
"title":{
"type": "string",
"analyzer": "ik_complex"
},
"cover":{
"type": "string",
"analyzer": "ik_complex"
},
"summary":{
"type": "string",
"analyzer": "ik_complex"
},
"relate_account": {
"type": "integer"
},
"tag_ids":{
"type": "string",
"index": "not_analyzed"
},
"tag_names":{
"type": "string",
"index": "not_analyzed"
},
"tag_count": {
"type": "integer"
}
}
}
}
\ No newline at end of file
... ...
... ... @@ -84,5 +84,18 @@
<appenderClass>com.yohomars.search.index.appender.impls.SocialUserIndexAppender</appenderClass>
<mappingFile>esmapping/social_user.json</mappingFile>
</index>
<index>
<name>content</name>
<properties>
<property key="number_of_shards" value="1"/>
<property key="number_of_replicas" value="0"/>
<property key="refresh_interval" value="10s"/>
<property key="translog.flush_threshold_ops" value="5000"/>
</properties>
<builderClass>com.yohomars.search.index.builder.impls.ContentIndexBuilder</builderClass>
<appenderClass>com.yohomars.search.index.appender.impls.ContentIndexAppender</appenderClass>
<mappingFile>esmapping/content.json</mappingFile>
</index>
</client>
</IndexConfigs>
... ...
... ... @@ -23,5 +23,6 @@ datasources:
password: ${jdbc.mysql.yhnow.password}
daos:
- com.yoho.search.dal.SocialUserDetailMapper
- com.yoho.search.dal.ContentMapper
readOnlyInSlave: true
\ No newline at end of file
... ...
... ... @@ -84,5 +84,18 @@
<appenderClass>com.yohomars.search.index.appender.impls.SocialUserIndexAppender</appenderClass>
<mappingFile>esmapping/social_user.json</mappingFile>
</index>
<index>
<name>content</name>
<properties>
<property key="number_of_shards" value="1"/>
<property key="number_of_replicas" value="${yohomarssearch.index.number_of_replicas}"/>
<property key="refresh_interval" value="${yohomarssearch.index.refresh_interval}"/>
<property key="translog.flush_threshold_ops" value="${yohomarssearch.index.translog.flush_threshold_ops}"/>
</properties>
<builderClass>com.yohomars.search.index.builder.impls.ContentIndexBuilder</builderClass>
<appenderClass>com.yohomars.search.index.appender.impls.ContentIndexAppender</appenderClass>
<mappingFile>esmapping/content.json</mappingFile>
</index>
</client>
</IndexConfigs>
... ...