seoModel.js 6.41 KB
'use strict';
const api = global.yoho.API;
const redis = global.yoho.redis;
const rp = require('request-promise');
const serviceApi = global.yoho.ServiceAPI;
const Promise = require('bluebird');
const co = Promise.coroutine;
const _ = require('lodash');
const logger = global.yoho.logger;
const helper = global.yoho.helpers;
const config = require('../config/config');
const qs = require('querystring');


/**
 * redis multi command
 */
const multiAsync = (multi)=>{
    return multi.execAsync().then(function(res) {
        return res;
    });
};

const baiduUrls = {
    urls: 'http://data.zz.baidu.com/urls',
    update: 'http://data.zz.baidu.com/update',
    del: 'http://data.zz.baidu.com/del'
};

const siteUrls = {
    pcProduct: {
        site: 'https://item.yohobuy.com',
        url: []
    },
    pcGuang: {
        site: 'https://guang.yohobuy.com',
        url: []
    },
    mProduct: {
        site: 'https://m.yohobuy.com',
        url: []
    },
    mGuang: {
        site: 'https://guang.m.yohobuy.com',
        url: [],
        type: 'mip'
    }

};



// 配置
const redisKey = {
    keywordsList: 'keywords_mana_list' // 关键词列表
};

/**
 * 将链接推送到百度站长
 * @param params object {site: 'https://www.yohobuy.com', type: 'mip'} 默认不需要type
 * @param urls
 */
const sendUrlsToBaidu = (params, urls) => {
    let paramsDef = {
        token: config.baiduToken
    };

    // 过滤无效的参数
    _.forEach(params, (val, key) => {
        if (!val) {
            delete params[key];
        }
    });

    qs.escape = (str) => {
        return str;
    };

    let options = {
        url: `${baiduUrls.urls}?${qs.stringify(Object.assign(paramsDef, params), null, null, {})}`,
        headers: {
            'Content-Type': 'text/plain'
        },
        method: 'post',
        form: urls.join('\n'),
        json: true,
        timeout: 10000,
        gzip: true
    };

    return rp(options).then(result => {
        logger.info(Object.assign(params, result, {length: urls.length}));
    });
};

/**
 * 获取最新1000条商品详情链接和逛详情链接
 */
const getUrls = () => {
    let apiArr = [api.get('', {method: 'web.product.bdPromotion'}),
        serviceApi.get('/guang/api/v2/article/getLastArticleList', {limit: 100})];

    return api.all(apiArr).spread((productData, articleData) => {

        _.forEach(_.get(productData, 'data', {}), value => {
            siteUrls.pcProduct.url.push('https:' + helper.urlFormat(`/${value.id}.html`, null, 'item'));
            siteUrls.mProduct.url.push('https:' + helper.urlFormat(`/product/${value.id}.html`, null, 'm'));
        });

        _.forEach(_.get(articleData, 'data.artList', {}), value => {
            siteUrls.pcGuang.url.push('https:' + helper.urlFormat(`/${value.articleId}.html`, null, 'guang'));
            siteUrls.mGuang.url.push('https:' + helper.urlFormat(`/mip/guang/info/${value.articleId}.html`
        , null, 'guang.m'));
        });

        return siteUrls;
    });
};

/**
 * 发送最新商品详情1000条和逛详情100条推送到相应的站点域名(pc和wap)
 */
const sendUrls = () => {

    co(function*() {
        // 获取pc/wap的商品详情和逛的链接
        let sendArr = [],
            urls = yield getUrls();

        _.forEach(urls, value => {
            sendArr.push(sendUrlsToBaidu({site: value.site, type: value.type}, value.url));
        });

        // 推送url
        api.all(sendArr);
    })();
};



/**
 * 调用接口建议词
 */
const getKeywordsApi = (page, limit) => {
    let params = {
        page: page || 1,
        limit: limit || 1000,
        method: 'web.search.suggestList'
    };

    return api.get('', params);
};

/**
 * 关键词同步到redis
 */
const synchronousKeywords = () => {

    return getKeywordsApi(1, 1).then(res => {
        let start = 0,
            page = 1,
            intervalTime = 1000, // 循环调用的时间间隔
            limit = 1000, // 每次请求接口关键词数量
            total = _.get(res, 'data.total', 0);

            // 接口调用失败
        if (total <= 0) {
            console.log('no data');
            return;
        }

            // 循环遍历接口关键词写入redis
        let interval = setInterval(() => {
            if (start > total) {
                clearInterval(interval);
            }

            getKeywordsApi(page, limit).then(result => {
                let multi = redis.multi();

                start += limit;
                page++;
                console.log(page);
                _.forEach(_.get(result, 'data.suggest_list', []), value => {
                    let key = `keywords_mana:${value.keyword}`;

                    multi.set(key, value.keyword);
                    multi.lrem('keywords_mana_list', 1, key).lpush('keywords_mana_list', key);
                });

                multiAsync(multi);

            }).catch(()=>{
                clearInterval(interval);
            });

        }, intervalTime);

    });
};

/**
 * 查询 redis中 关键词
 * @type {{getKeyWordsUrl}}
 */

const getRedisKeywords = (start, end) => {
    return redis.lrangeAsync(redisKey.keywordsList, start, end).then(res => {
        let urls = {pc: [], wap: []};

        _.forEach(res, keyword => {
            let buff = new Buffer(keyword).toString('hex').toUpperCase();

            urls.pc.push(`https://www.yohobuy.com/so/${buff}.html`);
            urls.wap.push(`https://m.yohobuy.com/so/${buff}.html`);
        });

        return urls;
    });
};

/**
 * 推送url
 */
const sendKeywordsUrls = () => {
    return redis.llenAsync(redisKey.keywordsList).then(total => {
        console.log(total);
        if (total <= 0) {
            return;
        }

        let start = 0,
            intervalTime = 1000, // 循环调用的时间间隔
            count = 1000;

        let interval = setInterval(() => {
            if (start >= total) {
                clearInterval(interval);
            }

            getRedisKeywords(start, start + count).then(urls => {
                console.log(urls);

                // 发送到百度
                // sendUrlsToBaidu({site: 'https://www.yohobuy.com'}, urls.pc);
                // sendUrlsToBaidu({site: 'https://m.yohobuy.com'}, urls.wap);
            }).catch(() => {
                clearInterval(interval);
            });

            start += count;

        }, intervalTime);

        return [];

    });
};

module.exports = {
    sendUrls,
    synchronousKeywords,
    sendKeywordsUrls
};