# -*- coding:utf-8 -*-

#导入json模块，将json字符串转换为python字典

import json

from collections import defaultdict

from collections import Counter

from pandas import DataFrame, Series

import pandas as pd

import numpy as np

import matplotlib.pyplot as plt

path = "E:/Programming/Python/PythonDataAnalysis/datasets/usagov_bitly/example.txt"

#list comprehension

records = [json.loads(line) for line in open(path)]

#对时区计数, 同时保证tz必须在records中

time_zones = [rec['tz'] for rec in records if 'tz' in rec.keys()]

#--------------方法1:------------

#时区计数

def get_counts(sequence):

	counts = {}

	for x in sequence:

		if x in counts:

			counts[x] += 1

		else:

			counts[x] = 1

	return counts

#取得前n个最常使用的时区

def top_counts(count_dict,n = 10):

	value_key_pairs = [(count,tz) for tz, count in count_dict.items()]

	value_key_pairs.sort()

	return value_key_pairs[-n:]

counts = get_counts(time_zones)

print(counts)

top_counts = top_counts(counts)

print(top_counts)

#--------------方法2:------------

def get_counts2(sequence):

	counts = defaultdict(int)

	for x in sequence:

		counts[x] += 1

	return counts

#--------------方法3:------------

#引入collections的Counter对象

def get_counts3(time_zones,n=10):

	counts = Counter(time_zones)

	return counts.most_common(n)

top_counts3 = get_counts3(time_zones,10)

print(top_counts3)

#--------------方法3:------------

#用pandas对时区进行计数

#将records转换为DataFrame对象

frame = DataFrame(records)

#frame['tz']返回的对象有一个value_counts方法

tz_counts = frame['tz'].value_counts()

print(tz_counts[:10])

#fillna()函数填补空缺值NA

clean_tz = frame['tz'].fillna("Missing")

print(clean_tz)

#空字符串为Unknown

clean_tz[clean_tz == ''] = "Unknown"

tz_counts = clean_tz.value_counts()

print(tz_counts[:10])

#利用counts的plot方法

tz_counts[:10].plot(kind = "barh",rot=0)

plt.show()

#用户浏览器分析

results = Series([x.split()[0] for x in frame.a.dropna()])

#打印前8的浏览器

print(results.value_counts()[:8])

cframe = frame[frame.a.notnull()]

operating_system = np.where(cframe['a'].str.contains("Windows"),"Windows","Not Windows")

windows = 0

nonWindows = 0

for op in operating_system:

	if op == "Windows":

		windows += 1

	else:

		nonWindows += 1

print("windows:",windows,"nonWindows:",nonWindows)

#使用windows/nonwindows给时区分组

by_tz_os = cframe.groupby(['tz',operating_system])

agg_counts = by_tz_os.size().unstack().fillna(0)

print(agg_counts[:10])

#选取最常见的时区

indexer = agg_counts.sum(1).argsort()

print(indexer)

count_subset = agg_counts.take(indexer)[-10:]

print(count_subset)

#绘制windows/nonwindows 堆叠条形图

count_subset.plot(kind="barh",stacked=True)

#不加这句语句，在Ipython中可以显示但是脚本运行不显示

plt.show()

#规范化

normed_subset = count_subset.div(count_subset.sum(1),axis = 0)

normed_subset.plot(kind = "barh",stacked=True)

plt.show()

# -*- coding: utf-8 -*-

import pandas as pd

import os

#数据读取，读成3个表

path = 'E:/Programming/Python/PythonDataAnalysis/datasets/movielens/'

unames = ['user_id','gender','age','occupation','zip']

upath = os.path.join(path,'users.dat')

users = pd.read_table(upath,sep = "::",header=None,names=unames,engine='python')

rnames = ['user_id',"movie_id","rating","timestamp"]

ratings = pd.read_table(path+'ratings.dat',sep = "::",header=None,names=rnames,engine='python')

mnames = ['movie_id','title','genres']

movies = pd.read_table(path+'movies.dat',sep ="::",header=None,names=mnames,engine='python')

#数据表整合

data = pd.merge(pd.merge(ratings,users),movies)

print(data[:10])

print(data.ix[0])

#按性别计算每部电影的得分,index 中是标签，columns中是列标签

mean_ratings = data.pivot_table('rating',index = 'title',columns = "gender",aggfunc='mean')

print(mean_ratings[:10])

#过滤掉评分不足250条的电影

ratings_by_title = data.groupby('title').size()

print(ratings_by_title[:10])

active_titles = ratings_by_title[ratings_by_title >= 250]

print(active_titles)

#按照评论>=250的index筛选

mean_ratings = mean_ratings.ix[active_titles.index]

top_female_ratings = mean_ratings.sort_index(by='F',ascending=False)

print(top_female_ratings[:10])

#计算男性女性得分分歧最大的电影

mean_ratings['diff'] = mean_ratings['M'] - mean_ratings['F']

sorted_by_diff = mean_ratings.sort_index(by = 'diff')

#分歧最大且女性更喜欢的电影

print(sorted_by_diff[:15])

#对结果反序取出前15行，男性观众更喜欢的电影

print(sorted_by_diff[::-1][:15])

#分歧最大的电影，计算方差或者标准差

rating_std_by_title = data.groupby('title')['rating'].std()

#使用active_title进行过滤

rating_std_by_title = rating_std_by_title.ix[active_titles]

rating_std_by_title.order(ascending=False)

print(rating_std_by_title[:15])

# -*- coding:utf-8 -*-

import pandas as pd

import matplotlib.pyplot as plt

import numpy as np

path = 'E:/Programming/Python/PythonDataAnalysis/datasets/babynames/'

names1880 = pd.read_csv(path+'yob1880.txt',names = ['name','sex','births'],engine='python')

#按照sex对数据进行简单分组

names1880.groupby('sex').births.sum()

#将单个文件中的数据整合到一个数据表中

years = range(1880,2011)

pieces = []

columns = ['name','sex','birth']

for year in years:

	subpath = 'yob%d.txt' % year

	frame = pd.read_csv(path+subpath,names = columns)

	frame['year'] = year

	pieces.append(frame)

names = pd.concat(pieces,ignore_index = True)

#使用pivot_table()函数进行聚合

total_births = names.pivot_table('birth',index = 'year',columns = 'sex',aggfunc = sum)

print(total_births.tail())

#插入prop列存放指定的婴儿数相对于总出生数的比例

def add_prop(group):

	births = group.birth.astype(float)

	group['prop'] = births/births.sum()

	return group

names = names.groupby(['year','sex']).apply(add_prop)

#取出每个sex/year组合的前1000个名字

def get_top1000(group):

	return group.sort_values(by='birth',ascending=False)[1:1000]

grouped = names.groupby(['year','sex'])

top1000 = grouped.apply(get_top1000)

#接下来的'命名趋势'分析针对这top1000个数据集

#取出男性

boys = top1000[top1000.sex == 'M']

#取出女性

girls = top1000[top1000.sex == 'F']

total_births = top1000.pivot_table('birth',index = 'year',columns = 'name',aggfunc = sum)

subset = total_births[['John','Harry','Mary','Marilyn']]

subset.plot(subplots = True,figsize = (12,10),grid=False,title = "Number of births per year")

plt.show()

#观察名字多样性变化

table = top1000.pivot_table('prop',index = 'year',columns = 'sex',aggfunc = sum)

table.plot(title = "sum of table1000.prop by year and sex",yticks = np.linspace(0,1.2,13),xticks = range(1880,2020,10))

plt.show()

# 名字最后一个字母的变化