# -*- coding:utf-8 -*-
# item_mongodb.py

import redis

import pymongo

import json

def main():

    redis_connect = redis.StrictRedis(host='127.0.0.1', port=6379, db=0)

    mongo_connect = pymongo.MongoClient(host='127.0.0.1', port=27017)

    db = mongo_connect['sina']

    collection = db['sina_articles']

    while True:

        source, data = redis_connect.blpop(['sina:items'])

        item = json.loads(data)

        collection.insert(item)

if __name__ == '__main__':

    main()

# -*- coding:utf-8 -*-

# item_mysql.py

import MySQLdb

import mysql.connector

import redis

import json

def main():

    redis_connect = redis.StrictRedis(host='localhost', password='', port=6379, db=0)

    mysql_connect = mysql.connector.connect(host='127.0.0.1', port=3306, user='root', passwd='', database='sina', use_unicode=True)

    while redis_connect.exists('sina:items'):

        source, data = redis_connect.blpop('sina:items')

        item = json.loads(data, encoding='utf-8')

        print (item)

        cur = mysql_connect.cursor()

        cur.execute(

            'insert into sina_articles(parent_title, parent_url, sub_title, sub_url, sub_path, file_url, '

            'file_title, file_content, crawled, spider) values(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)',

            (item['parent_title'], item['parent_url'], item['sub_title'], item['sub_url'], item['sub_path'],

             item['file_url'], item['file_title'], item['file_content'], item['crawled'], item['spider']))

        mysql_connect.commit()

        cur.close()

    mysql_connect.close()

if __name__ == "__main__":

    main()

# -*- coding:utf-8 -*-

# redis数据库 到 Excel

import redis

import json

from openpyxl import Workbook

def main():

    # redis_connect = redis.StrictRedis(host='localhost', port=6379, db=0)

    redis_connect = redis.StrictRedis(host='localhost', password='', port=6379, db=0)

    a = 1

    while redis_connect.exists('hungry:items'):

        source, data = redis_connect.blpop('hungry:items')

        item = json.loads(data, encoding='utf-8')

        print (item)

        a =a + 1

        ws.append([item['id'],

                   item['classification'],

                   item['shopName'],

                   item['address'],

                   item['foodInformation'][0]['rating'],

                   item['foodInformation'][0]['price'],

                   int(item['foodInformation'][0]['sale']),

                   int(item['foodInformation'][0]['Evaluation']),

                   item['foodInformation'][0]['foodName'],

                   ])

    wb.save('nba.xlsx')  # 保存文件

    print ('a:', a)

if __name__ == "__main__":

    wb = Workbook()  # class实例化

    ws = wb.active  # 激活工作表

    ws.title = "New Shit"

    title_list = ['id', 'classification', 'shopName', 'address', 'rating', 'price', 'Evaluation', 'foodName', 'sale']

    ws['A1'] = '饿了么'

    ws.append(title_list)  # 添加一行数据

    main()