mongo2csv
2024-08-24 11:31:59
#!/usr/bin/env python3 import pymongo
import datetime
import os yesterday = str(datetime.date.today() - datetime.timedelta(days=1)) def mongo2csv(db_name, ts_string=yesterday, column_handle=None, column_delete=None):
''' :param db_name: 表名
:param ts_string: 日期
:param column_handle: 需要处理的列,
格式{k:v} ,
k:需要处理的列,
v:1.数字,即列表索引,取列表的第几个元素
2.字符串,即字典key,取子字典的某个value
3.join ,即需要将列表转为字符串 :param column_delete: 需要删除的列
:return:
'''
print('starting...')
host = '127.0.0.1'
port = 5600
user = 'root'
password = ''
url = 'mongodb://' + user + ':' + password + '@' + host + ':' + str(port) + '/'
client = pymongo.MongoClient(url)
db = client.rental
collection = db[db_name]
select_condition = {'_id': 0, 'ts': 0}
if not column_delete:
column_delete = []
if not column_handle:
column_handle = {}
if len(column_delete) > 0:
for temp in column_delete:
select_condition[temp] = 0
title_dic = collection.find_one({'ts_string': ts_string}, select_condition)
title = sorted(title_dic.keys())
con = collection.find({'ts_string': ts_string}, select_condition)
file_name = db_name + '_' + yesterday + '.csv'
with open(file_name, 'a', encoding='utf8') as f:
f.write(','.join(title) + '\n')
for item in con:
for k, v in column_handle.items():
item[k] = item[k][v] if v != 'join' else (
'|'.join([str(x) for x in item[k]]) if isinstance(item[k], list) else item[k])
f.write(','.join([str(item[x]) for x in title]) + '\n') if __name__ == '__main__':
mongo2csv('lianjia_detail')
mongo2csv('mogu_detail', column_handle={'metroInfo': 0, 'rentType': 'value'})
mongo2csv('qingke_detail', column_handle={})
mongo2csv('xiangyu_detail', column_delete=['endDate','vacantStartDate','tabList','vacantEndDate'])
mongo2csv('ziru_detail', column_handle={'subway_line_code': 'join', 'subway_station_code': 'join'})
print(os.system('wc -l *.csv'))
最新文章
- c#解析xml
- PKU 1003解题
- 2015baidu复赛2 连接的管道(mst &;&; 优先队列prim)
- Knockout 新版应用开发教程之创建view models与监控属性
- 无线路由器WDS简要
- Thread类详解
- C# HttpWebRequest类
- 51nod1240莫比乌斯函数
- UVA 11090 Going in Cycle!!
- unable to find valid certification path to requested target
- /调整button的title的位置
- vue+Element实现静态旅游网站
- Docker容器访问控制
- sql字符串包含单引号
- B. Creating the Contest(水题)
- BABOK概述
- raid1 raid2 raid5 raid6 raid10的优缺点和做各自raid需要几块硬盘
- python(字符串、列表、字典、元组、集合)的常用内置方法
- ORACLE 字段AES算法加密、解密
- 解决跨域No 'Access-Control-Allow-Origin' header is present on the requested resource.