yoho-search / yoho-search-consumer · Commits

Go to a project

Authored by wangnan9279
Commit a04e7e31132872820d391f3169c9c0c36d77318d 2 parents 000fef43 957cadaa

Merge branch '0920' into 0930

Inline Side-by-side
Showing 5 changed files with 36 additions and 25 deletions
@@ -4,6 +4,8 @@ @@ -4,6 +4,8 @@
4 <resultMap id="BaseResultMap" type="com.yoho.search.dal.model.CsRecallConfigCommon"> 4 <resultMap id="BaseResultMap" type="com.yoho.search.dal.model.CsRecallConfigCommon">
5 <id column="id" property="id" jdbcType="INTEGER"/> 5 <id column="id" property="id" jdbcType="INTEGER"/>
6 <result column="config_page" property="configPage" jdbcType="INTEGER"/> 6 <result column="config_page" property="configPage" jdbcType="INTEGER"/>
  7 + <result column="order_type" property="orderType" jdbcType="INTEGER"/>
  8 + <result column="ab_type" property="abType" jdbcType="INTEGER"/>
7 <result column="config_type" property="configType" jdbcType="VARCHAR"/> 9 <result column="config_type" property="configType" jdbcType="VARCHAR"/>
8 <result column="description" property="description" jdbcType="VARCHAR"/> 10 <result column="description" property="description" jdbcType="VARCHAR"/>
9 <result column="size" property="size" jdbcType="INTEGER"/> 11 <result column="size" property="size" jdbcType="INTEGER"/>
@@ -11,37 +13,38 @@ @@ -11,37 +13,38 @@
11 <result column="interval" property="interval" jdbcType="INTEGER"/> 13 <result column="interval" property="interval" jdbcType="INTEGER"/>
12 <result column="interval_limit" property="intervalLimit" jdbcType="VARCHAR"/> 14 <result column="interval_limit" property="intervalLimit" jdbcType="VARCHAR"/>
13 <result column="insert_order" property="insertOrder" jdbcType="INTEGER"/> 15 <result column="insert_order" property="insertOrder" jdbcType="INTEGER"/>
14 - <result column="ab_type" property="abType" jdbcType="INTEGER"/>  
15 <result column="update_time" property="updateTime" jdbcType="TIMESTAMP"/> 16 <result column="update_time" property="updateTime" jdbcType="TIMESTAMP"/>
16 </resultMap> 17 </resultMap>
17 18
18 <sql id="Base_Column_List"> 19 <sql id="Base_Column_List">
19 - id, config_page, config_type, description, `size`, size_limit, `interval`, interval_limit, insert_order, ab_type, update_time 20 + id, config_page, order_type, ab_type, config_type, description, `size`, size_limit, `interval`, interval_limit, insert_order , update_time
20 </sql> 21 </sql>
21 22
22 <insert id="insert" parameterType="com.yoho.search.dal.model.CsRecallConfigCommon" timeout="20000"> 23 <insert id="insert" parameterType="com.yoho.search.dal.model.CsRecallConfigCommon" timeout="20000">
23 insert ignore into cs_recall_config_common ( 24 insert ignore into cs_recall_config_common (
24 id, 25 id,
25 config_page, 26 config_page,
  27 + order_type,
  28 + ab_type,
26 config_type, 29 config_type,
27 description, 30 description,
28 `size`, 31 `size`,
29 size_limit, 32 size_limit,
30 `interval`, 33 `interval`,
31 interval_limit, 34 interval_limit,
32 - insert_order,  
33 - ab_type ) 35 + insert_order)
34 values ( 36 values (
35 #{id,jdbcType=INTEGER}, 37 #{id,jdbcType=INTEGER},
36 #{configPage,jdbcType=INTEGER}, 38 #{configPage,jdbcType=INTEGER},
  39 + #{orderType,jdbcType=INTEGER},
  40 + #{abType,jdbcType=INTEGER},
37 #{configType,jdbcType=VARCHAR}, 41 #{configType,jdbcType=VARCHAR},
38 #{description,jdbcType=VARCHAR}, 42 #{description,jdbcType=VARCHAR},
39 #{size,jdbcType=INTEGER}, 43 #{size,jdbcType=INTEGER},
40 #{sizeLimit,jdbcType=VARCHAR}, 44 #{sizeLimit,jdbcType=VARCHAR},
41 #{interval,jdbcType=INTEGER}, 45 #{interval,jdbcType=INTEGER},
42 #{intervalLimit,jdbcType=VARCHAR}, 46 #{intervalLimit,jdbcType=VARCHAR},
43 - #{insertOrder,jdbcType=INTEGER},  
44 - #{abType,jdbcType=INTEGER} 47 + #{insertOrder,jdbcType=INTEGER}
45 ) 48 )
46 </insert> 49 </insert>
47 50
@@ -51,6 +54,12 @@ @@ -51,6 +54,12 @@
51 <if test="configPage != null"> 54 <if test="configPage != null">
52 config_page = #{configPage,jdbcType=INTEGER}, 55 config_page = #{configPage,jdbcType=INTEGER},
53 </if> 56 </if>
  57 + <if test="orderType != null">
  58 + order_type = #{orderType,jdbcType=INTEGER},
  59 + </if>
  60 + <if test="abType != null">
  61 + ab_type = #{abType,jdbcType=INTEGER},
  62 + </if>
54 <if test="configType != null"> 63 <if test="configType != null">
55 config_type = #{configType,jdbcType=VARCHAR}, 64 config_type = #{configType,jdbcType=VARCHAR},
56 </if> 65 </if>
@@ -70,10 +79,7 @@ @@ -70,10 +79,7 @@
70 interval_limit = #{intervalLimit,jdbcType=VARCHAR}, 79 interval_limit = #{intervalLimit,jdbcType=VARCHAR},
71 </if> 80 </if>
72 <if test="insertOrder != null"> 81 <if test="insertOrder != null">
73 - insert_order = #{insertOrder,jdbcType=VARCHAR},  
74 - </if>  
75 - <if test="abType != null">  
76 - ab_type = #{abType,jdbcType=VARCHAR}, 82 + insert_order = #{insertOrder,jdbcType=INTEGER},
77 </if> 83 </if>
78 </set> 84 </set>
79 where id = #{id,jdbcType=INTEGER} 85 where id = #{id,jdbcType=INTEGER}
@@ -146,16 +146,21 @@ public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEvent @@ -146,16 +146,21 @@ public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEvent
146 */ 146 */
147 private int genNumberOfReplicas(IElasticsearchClient client, Map<String, String> indexSettings) { 147 private int genNumberOfReplicas(IElasticsearchClient client, Map<String, String> indexSettings) {
148 try { 148 try {
149 - int numberOfDataNodes = client.getClusterHealthResponse().getNumberOfDataNodes();  
150 - if (numberOfDataNodes <= 1) { 149 + //获取数据节点机器数
  150 + int dataNodesCount = client.getClusterHealthResponse().getNumberOfDataNodes();
  151 + if (dataNodesCount <= 1) {
151 return 0; 152 return 0;
152 } 153 }
  154 + //获取配置的复制分片数
153 String numberOfReplicas = MapUtils.getString(indexSettings, "number_of_replicas");//配置的复制分片数 155 String numberOfReplicas = MapUtils.getString(indexSettings, "number_of_replicas");//配置的复制分片数
154 if (!"auto".equalsIgnoreCase(numberOfReplicas)) { 156 if (!"auto".equalsIgnoreCase(numberOfReplicas)) {
155 return Integer.valueOf(numberOfReplicas); 157 return Integer.valueOf(numberOfReplicas);
156 } 158 }
157 - //获取索引配置的主分片数  
158 - return Math.max(1, numberOfDataNodes - 1); 159 + // 获取配置的主分片数
  160 + int numberOfPrimary = MapUtils.getIntValue(indexSettings, "number_of_shards");//配置的主分片数
  161 + // 动态计算复制分片数【1个主分片就是机器数-1 ， 4个主分片，4台机器就是 0，然后取1 】
  162 + int number_of_replicas = dataNodesCount/numberOfPrimary - 1;
  163 + return Math.max(1, number_of_replicas);
159 } catch (Exception e) { 164 } catch (Exception e) {
160 INDEX_REBUILD_LOG.error(e.getMessage(), e); 165 INDEX_REBUILD_LOG.error(e.getMessage(), e);
161 return 0; 166 return 0;
@@ -206,7 +211,7 @@ public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEvent @@ -206,7 +211,7 @@ public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEvent
206 indexSettings.put("merge.policy.expunge_deletes_allowed", 20d); // 删除了的文档数在一个segment里占的百分比，默认是10 211 indexSettings.put("merge.policy.expunge_deletes_allowed", 20d); // 删除了的文档数在一个segment里占的百分比，默认是10
207 indexSettings.put("merge.policy.floor_segment", "2mb"); // ES会避免产生很小size的segment，小于这个阈值的所有的非常小的segment都会做merge,默认是2MB. 212 indexSettings.put("merge.policy.floor_segment", "2mb"); // ES会避免产生很小size的segment，小于这个阈值的所有的非常小的segment都会做merge,默认是2MB.
208 indexSettings.put("merge.policy.max_merge_at_once", 10); // 一次最多只操作多少个segments，默认是10. 213 indexSettings.put("merge.policy.max_merge_at_once", 10); // 一次最多只操作多少个segments，默认是10.
209 - indexSettings.put("merge.policy.max_merged_segment", "2gb"); // 超过多大size的segment不会再做merge，默认是5g. 214 + indexSettings.put("merge.policy.max_merged_segment", "500mb"); // 超过多大size的segment不会再做merge，默认是5g.
210 indexSettings.put("merge.policy.segments_per_tier", 10d); // 每个tier允许的segement 数，注意这个数要大于上面的at_once数，否则这个值会先于最大可操作数到达，就会立刻做merge，这样会造成频繁 215 indexSettings.put("merge.policy.segments_per_tier", 10d); // 每个tier允许的segement 数，注意这个数要大于上面的at_once数，否则这个值会先于最大可操作数到达，就会立刻做merge，这样会造成频繁
211 216
212 client.createIndex(newRealIndexName, type, indexSettings, mappingContent); 217 client.createIndex(newRealIndexName, type, indexSettings, mappingContent);
@@ -10,11 +10,17 @@ @@ -10,11 +10,17 @@
10 "id": { 10 "id": {
11 "type": "integer" 11 "type": "integer"
12 }, 12 },
  13 + "configPage": {
  14 + "type": "integer"
  15 + },
  16 + "orderType": {
  17 + "type": "integer"
  18 + },
13 "configType": { 19 "configType": {
14 "type": "keyword" 20 "type": "keyword"
15 }, 21 },
16 - "configPage": {  
17 - "type": "integer" 22 + "description": {
  23 + "type": "keyword"
18 }, 24 },
19 "size": { 25 "size": {
20 "type": "integer" 26 "type": "integer"
@@ -28,15 +34,9 @@ @@ -28,15 +34,9 @@
28 "intervalLimit": { 34 "intervalLimit": {
29 "type": "keyword" 35 "type": "keyword"
30 }, 36 },
31 - "description": {  
32 - "type": "keyword"  
33 - },  
34 "insertOrder": { 37 "insertOrder": {
35 "type": "integer" 38 "type": "integer"
36 }, 39 },
37 - "abType": {  
38 - "type": "integer"  
39 - },  
40 "updateTime": { 40 "updateTime": {
41 "type": "date" 41 "type": "date"
42 } 42 }
@@ -80,7 +80,7 @@ @@ -80,7 +80,7 @@
80 <index> 80 <index>
81 <name>productindex</name> 81 <name>productindex</name>
82 <properties> 82 <properties>
83 - <property key="number_of_shards" value="2"/> 83 + <property key="number_of_shards" value="4"/>
84 <property key="number_of_replicas" value="auto"/> 84 <property key="number_of_replicas" value="auto"/>
85 <property key="refresh_interval" value="10s"/> 85 <property key="refresh_interval" value="10s"/>
86 <property key="translog.flush_threshold_size" value="100mb"/> 86 <property key="translog.flush_threshold_size" value="100mb"/>
@@ -80,7 +80,7 @@ @@ -80,7 +80,7 @@
80 <index> 80 <index>
81 <name>productindex</name> 81 <name>productindex</name>
82 <properties> 82 <properties>
83 - <property key="number_of_shards" value="2"/> 83 + <property key="number_of_shards" value="4"/>
84 <property key="number_of_replicas" value="auto"/> 84 <property key="number_of_replicas" value="auto"/>
85 <property key="refresh_interval" value="${search.index.refresh_interval}"/> 85 <property key="refresh_interval" value="${search.index.refresh_interval}"/>
86 <property key="translog.flush_threshold_size" value="${search.index.translog.flush_threshold_size}"/> 86 <property key="translog.flush_threshold_size" value="${search.index.translog.flush_threshold_size}"/>