《利用Python进行数据分析》读书笔记_2_第一二章_准备工作及引言

字数统计: 1.9k字   |   阅读时长≈ 10分

第一章 准备工作

这一章节主要是环境构建工作。

遇到问题，无法安装pandas包，pip也无法安装；
命令sudo easy_install xxx，怀疑是网速原因，超时了。
所以直接下载了zip包。。补充：之后在公司网速可以的情况下由下载下来了，安装了pip工具。

本书用到的数据地址

第二章 引言

来自bit.ly的1.usa.gov数据

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
#!/usr/bin/python
#-*- coding:utf-8 -*-

# 读取文件中一行
path = '../pydata-book-master/ch02/usagov_bitly_data2012-03-16-1331923249.txt'
print open(path).readline()
# out: { "a": "Mozilla\/5.0 (Windows NT 6.1; WOW64) AppleWebKit\/535.11 (KHTML, like Gecko) Chrome\/17.0.963.78 Safari\/535.11", "c": "US", "nk": 1, "tz": "America\/New_York", "gr": "MA", "g": "A6qOVH", "h": "wfLQtf", "l": "orofrog", "al": "en-US,en;q=0.8", "hh": "1.usa.gov", "r": "http:\/\/www.facebook.com\/l\/7AQEFzjSi\/1.usa.gov\/wfLQtf", "u": "http:\/\/www.ncbi.nlm.nih.gov\/pubmed\/22415991", "t": 1331923247, "hc": 1331822918, "cy": "Danvers", "ll": [ 42.576698, -70.954903 ] }

import json
# 列表推导式，生成的是一组Python字典
record = [json.loads(line) for line in open(path)]
print record[0]
# out：{u'a': u'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.78 Safari/535.11', u'c': u'US', u'nk': 1, u'tz': u'America/New_York', u'gr': u'MA', u'g': u'A6qOVH', u'h': u'wfLQtf', u'cy': u'Danvers', u'l': u'orofrog', u'al': u'en-US,en;q=0.8', u'hh': u'1.usa.gov', u'r': u'http://www.facebook.com/l/7AQEFzjSi/1.usa.gov/wfLQtf', u'u': u'http://www.ncbi.nlm.nih.gov/pubmed/22415991', u't': 1331923247, u'hc': 1331822918, u'll': [42.576698, -70.954903]}
# 既然已经生成了字典项，就可以通过键访问值
print record[0]['ll']
# out: [42.576698, -70.954903]
# 单引号前面的u标识unicode；
print record[0]['tz']
# out: America/New_York  # 时区是字符串对象形式，而非打印形式

纯Python时区计数

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
time_zones = [rec['tz'] for rec in record if 'tz' in rec]
print time_zones[:10]

# 对时区进行计数
def get_counts(sequence):
    counts = {}
    for x in sequence:
        if x in counts:
            counts[x] += 1
        else:
            counts[x] = 1
    return counts
# 使用标准库简化方法
from collections import defaultdict
def get_counts2(sequence):
    counts = defaultdict(int)
    for x in sequence:
        counts[x]+=1
    return counts
counts = get_counts(time_zones)
print counts['America/New_York']
# out: 1251
print len(time_zones)
# 3340
# 前10位的时区及计数
def top_counts(count_dict, n = 10):
    value_key_pairs = [(count, tz) for tz, count in count_dict.items()]
    value_key_pairs.sort()
    return value_key_pairs[-n:]
print top_counts(counts)
# out: [(33, u'America/Sao_Paulo'), (35, u'Europe/Madrid'), (36, u'Pacific/Honolulu'), (37, u'Asia/Tokyo'), (74, u'Europe/London'), (191, u'America/Denver'), (382, u'America/Los_Angeles'), (400, u'America/Chicago'), (521, u''), (1251, u'America/New_York')]
# 更简单的方式是collections.Counter类
from collections import Counter
counts1 = Counter(time_zones)
print counts1.most_common(10)
# out: [(u'America/New_York', 1251), (u'', 521), (u'America/Chicago', 400), (u'America/Los_Angeles', 382), (u'America/Denver', 191), (u'Europe/London', 74), (u'Asia/Tokyo', 37), (u'Pacific/Honolulu', 36), (u'Europe/Madrid', 35), (u'America/Sao_Paulo', 33)]

pandas对时区计数

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
# pandas时区计数
# DataFrame是pandas中最重要的数据结构，将数据处理成表格
from pandas import DataFrame, Series
import pandas as pd ;import numpy as np;import matplotlib.pyplot as plt
frame = DataFrame(record)
# print frame
# print frame['tz'][:10]
tz_counts = frame['tz'].value_counts()
# print tz_counts[:10]
# tz_counts[:10].plot(kind='barh', rot=0)
# 这里如不添加下句，图形闪退
# plt.show()
# print frame['a'][1]
result = Series([x.split()[0] for x in frame.a.dropna()])
# print result[:5]
# print result.value_counts()[:8]
cframe = frame[frame.a.notnull()]
operating_sys = np.where(cframe['a'].str.contains('Windows'),("Windows"),'Not Windows')
# print operating_sys[:5]
by_tz_os = cframe.groupby(['tz',operating_sys])
agg_counts = by_tz_os.size().unstack().fillna(0)
# print agg_counts[:5]
indexer = agg_counts.sum(1).argsort()
# print indexer[:5]
count_subset = agg_counts.take(indexer)[-10:]
# print count_subset
# count_subset.plot(kind='barh', stacked=True)
# plt.show()
normed_subset = count_subset.div(count_subset.sum(1),axis=0)
# normed_subset.plot(kind='barh', stached=True)
# plt.hold()
# plt.interactive(False)
# plt.show()

MovieLens 1M数据集

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
unames = ['user_id', 'gender', 'age', 'occupation', 'zip']
users = pd.read_table('../pydata-book-master/ch02/movielens/users.dat', sep='::', header=None, names=unames, engine='python')

rnames = ['user_id', 'movie_id', 'rating', 'timestamp']
ratings = pd.read_table('../pydata-book-master/ch02/movielens/ratings.dat', sep='::', header=None, names=rnames, engine='python')

mnames = ['movie_id', 'title', 'genres']
movies = pd.read_table('../pydata-book-master/ch02/movielens/movies.dat', sep='::', header=None, names=mnames, engine='python')

# print users[:5]
# print ratings[:5]
# print movies[:4]

# pandas的merge函数根据列名的重叠情况推断出哪些列是合并（或连接）键
data = pd.merge(pd.merge(ratings, users), movies)
# print data
# data.ix[0]

# 按性别计算每部电影平均分，使用pivot_table方法
mean_ratings = data.pivot_table('rating', index='title', columns='gender', aggfunc='mean')
# print mean_ratings[:5]
rating_by_title = data.groupby('title').size()
# print rating_by_title[:10]
# 过滤掉评分小于250
active_title = rating_by_title.index[rating_by_title>=250]
# print active_title
mean_ratings =mean_ratings.ix[active_title]
# print mean_ratings
# 按女性分数排序
top_female_rating = mean_ratings.sort_values(by='F', ascending=False)
# print top_female_rating[:4]

# 计算评分分歧；男女观众的分歧；方法是加上一列diff，做差;按diff排序，得出的结果是分歧 最大且女性更喜欢的电影。
mean_ratings['diff'] = mean_ratings['M'] - mean_ratings['F']
sorted_by_diff = mean_ratings.sort_values(by='diff')
# print sorted_by_diff[:10]
# 对行反序，并去除前十，则是男性更喜欢的电影
# print sorted_by_diff[::-1][:10]

# 根据电影名称分组的得分数据的标准差
rating_std_by_title = data.groupby('title')['rating'].std()
# 根据active_titles进行过滤
rating_std_by_title = rating_std_by_title.ix[active_title]
# 根据值对Series进行降序排列
rating_std_by_title.sort_value(ascending=False)[:5]
# AttributeError: 'Series' object has no attribute 'sort_value';按树上对order也不行，deprecated

1880–2010年间全美婴儿姓名

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
# 1880-2010年间全美婴儿姓名
# 用linux命令查看某个文件对前十行
# head -n 10 names/yob1880.txt
# 第一行：Mary,F,7065
# 数据由标准逗号隔开，可以用pandas.read_csv加载到DataFrame中
import pandas as pd;import matplotlib.pyplot as plt;import numpy as  np
# names1880 = pd.read_csv('../pydata-book-master/ch02/names/yob1880.txt', names=['name', 'sex', 'birthsNum'])
# print names1880
# print names1880.groupby('sex').birthsNum.sum()
# out: sex
#      F     90993
#      M    110493
#      Name: birthsNum, dtype: int64

# 将所有文件中到数据组装到一个DataFrame中，并加上一个year字段。使用pandas.concat
# 2010年是目前最后一个有效统计年度
years = range(1880, 2011)
pieces = []
columns = ['name', 'sex', 'birthsNum']
for year in years:
    path = '../pydata-book-master/ch02/names/yob%d.txt' % year
    frame = pd.read_csv(path, names=columns)

    frame['year'] = year
    pieces.append(frame)
# 将所有数据整合到单个DataFrame中;ignore_index必须置为False，不保留原始行号；concat默认按行整合
names = pd.concat(pieces, ignore_index=False)
# print names
# 聚合;DataFrame.pivot_table(y轴，x轴，多种类型-线条，每一年每种性别对出生总数)
total_birthsNum = names.pivot_table('birthsNum', index='year', columns='sex', aggfunc=sum)
# print total_birthsNum.tail()
# total_birthsNum.plot(title='Total birthsNum by sex and year')
# plt.show()

# 插入prop列，表示相对比例；由于birthsNum是整数，需要将分子分母转换成浮点数（Python3.x除外）
def add_prop(group):
    # 整数除法会向下圆整
    birthsNum = group.birthsNum.astype(float)
    group['prop'] = birthsNum/birthsNum.sum()
    return group
names = names.groupby(['year', 'sex']).apply(add_prop)
# print names
# 分组前应该进行有效性检查，用numpy.allclose=1判定所有prop之和是否为1
# print np.allclose(names.groupby(['year', 'sex']).prop.sum(), 1)
# out: True
# 为便于分析，取出前1000条按sex/year组合对数据
def get_top1000(group):
    return group.sort_index(by='birthsNum', ascending=False)[:1000]
grouped = names.groupby(['year', 'sex'])
top_1000 = grouped.apply(get_top1000)
# print top_1000
# 接下来针对这1000条数据进行分析

感到这一章就是先展示一下能干什么，用到的东西不熟悉，也只能照着敲敲；因此决定暂不跟着敲了；P41-P47跳过。

扫一扫，分享到微信

tag:

    缺失模块。
    1、请确保node版本大于6.2
    2、在博客根目录（注意不是yilia-plus根目录）执行以下命令：
    npm i hexo-generator-json-content --save

    3、在根目录_config.yml里添加配置： 

      jsonContent:
    meta: false
    pages: false
    posts:
      title: true
      date: true
      path: true
      text: false
      raw: false
      content: false
      slug: false
      updated: false
      comments: false
      link: false
      permalink: false
      excerpt: false
      categories: false
      tags: true