利用python进入数据分析之pandas的使用

导入相关库

In [2]:

from pandas import Series, DataFrameimport pandas as pdfrom __future__ import divisionfrom numpy.random import randnimport numpy as npimport osimport matplotlib.pyplot as pltnp.random.seed(12345)plt.rc('figure', figsize=(10, 6))from pandas import Series, DataFrameimport pandas as pdnp.set_printoptions(precision=4)

pandas的数据结构介绍

In [3]:

obj = Series([4, 7, -5, 3]) # 创建数组对象obj

Out[3]:

0    41    72   -53    3dtype: int64

In [4]:

obj.values

Out[4]:

array([ 4,  7, -5,  3], dtype=int64)

In [5]:

obj.index

Out[5]:

RangeIndex(start=0, stop=4, step=1)

In [6]:

obj2 = Series([4, 7, -5, 3], index=['d', 'b', 'a', 'c'])

In [7]:

obj2

Out[7]:

d    4b    7a   -5c    3dtype: int64

In [8]:

obj2.index

Out[8]:

Index([u'd', u'b', u'a', u'c'], dtype='object')

In [9]:

obj2['a']

Out[9]:

-5

In [10]:

obj2['d'] = 6obj2[['c', 'a', 'd']]

Out[10]:

c    3a   -5d    6dtype: int64

In [11]:

obj2[obj2 > 0]

Out[11]:

d    6b    7c    3dtype: int64

In [12]:

obj2 * 2

Out[12]:

d    12b    14a   -10c     6dtype: int64

In [13]:

np.exp(obj2)

Out[13]:

d     403.428793b    1096.633158a       0.006738c      20.085537dtype: float64

In [14]:

'b' in obj2

Out[14]:

True

In [15]:

'e' in obj2

Out[15]:

False

In [16]:

sdata = {'Ohio': 35000, 'Texas': 71000, 'Oregon': 16000, 'Utah': 5000}obj3 = Series(sdata)obj3

Out[16]:

Ohio      35000Oregon    16000Texas     71000Utah       5000dtype: int64

In [17]:

states = ['California', 'Ohio', 'Oregon', 'Texas']obj4 = Series(sdata, index=states)obj4

Out[17]:

California        NaNOhio          35000.0Oregon        16000.0Texas         71000.0dtype: float64

In [18]:

pd.isnull(obj4)  #检测是否缺失数据

Out[18]:

California     TrueOhio          FalseOregon        FalseTexas         Falsedtype: bool

In [19]:

pd.notnull(obj4)

Out[19]:

California    FalseOhio           TrueOregon         TrueTexas          Truedtype: bool

In [20]:

obj4.isnull()#检测是否缺失数据

Out[20]:

California     TrueOhio          FalseOregon        FalseTexas         Falsedtype: bool

In [21]:

obj3

Out[21]:

Ohio      35000Oregon    16000Texas     71000Utah       5000dtype: int64

In [22]:

obj4

Out[22]:

California        NaNOhio          35000.0Oregon        16000.0Texas         71000.0dtype: float64

In [23]:

obj3 + obj4

Out[23]:

California         NaNOhio           70000.0Oregon         32000.0Texas         142000.0Utah               NaNdtype: float64

In [24]:

obj4.name = 'population' # 设置名字obj4.index.name = 'state'# 设置索引名字obj4

Out[24]:

stateCalifornia        NaNOhio          35000.0Oregon        16000.0Texas         71000.0Name: population, dtype: float64

In [25]:

obj.index = ['Bob', 'Steve', 'Jeff', 'Ryan']obj

Out[25]:

Bob      4Steve    7Jeff    -5Ryan     3dtype: int64

DataFrame

In [26]:

data = {'state': ['Ohio', 'Ohio', 'Ohio', 'Nevada', 'Nevada'],        'year': [2000, 2001, 2002, 2001, 2002],        'pop': [1.5, 1.7, 3.6, 2.4, 2.9]}frame = DataFrame(data)

In [27]:

frame

Out[27]:

 popstateyear01.5Ohio200011.7Ohio200123.6Ohio200232.4Nevada200142.9Nevada2002

In [28]:

DataFrame(data, columns=['year', 'state', 'pop']) # 设置列索引

Out[28]:

 yearstatepop02000Ohio1.512001Ohio1.722002Ohio3.632001Nevada2.442002Nevada2.9

In [29]:

frame2 = DataFrame(data, columns=['year', 'state', 'pop', 'debt'],                   index=['one', 'two', 'three', 'four', 'five'])frame2

Out[29]:

 yearstatepopdebtone2000Ohio1.5NaNtwo2001Ohio1.7NaNthree2002Ohio3.6NaNfour2001Nevada2.4NaNfive2002Nevada2.9NaN

In [30]:

frame2.columns

Out[30]:

Index([u'year', u'state', u'pop', u'debt'], dtype='object')

In [31]:

frame2['state']

Out[31]:

one        Ohiotwo        Ohiothree      Ohiofour     Nevadafive     NevadaName: state, dtype: object

In [32]:

frame2.year

Out[32]:

one      2000two      2001three    2002four     2001five     2002Name: year, dtype: int64

In [33]:

frame2.ix['three'] # 通过ix，索引字段进行索引

D:\python2713\lib\anaconda_install\lib\site-packages\ipykernel_launcher.py:1: DeprecationWarning: .ix is deprecated. Please use.loc for label based indexing or.iloc for positional indexingSee the documentation here:http://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate_ix  """Entry point for launching an IPython kernel.

Out[33]:

year     2002state    Ohiopop       3.6debt      NaNName: three, dtype: object

In [34]:

frame2['debt'] = 16.5 # 列赋值frame2

Out[34]:

 yearstatepopdebtone2000Ohio1.516.5two2001Ohio1.716.5three2002Ohio3.616.5four2001Nevada2.416.5five2002Nevada2.916.5

In [35]:

frame2['debt'] = np.arange(5.)# 列赋值frame2

Out[35]:

 yearstatepopdebtone2000Ohio1.50.0two2001Ohio1.71.0three2002Ohio3.62.0four2001Nevada2.43.0five2002Nevada2.94.0

In [36]:

val = Series([-1.2, -1.5, -1.7], index=['two', 'four', 'five']) # 指定列赋值frame2['debt'] = valframe2

Out[36]:

 yearstatepopdebtone2000Ohio1.5NaNtwo2001Ohio1.7-1.2three2002Ohio3.6NaNfour2001Nevada2.4-1.5five2002Nevada2.9-1.7

In [37]:

frame2['eastern'] = frame2.state == 'Ohio'frame2

Out[37]:

 yearstatepopdebteasternone2000Ohio1.5NaNTruetwo2001Ohio1.7-1.2Truethree2002Ohio3.6NaNTruefour2001Nevada2.4-1.5Falsefive2002Nevada2.9-1.7False

In [38]:

del frame2['eastern']frame2.columns

Out[38]:

Index([u'year', u'state', u'pop', u'debt'], dtype='object')

In [39]:

pop = {'Nevada': {2001: 2.4, 2002: 2.9},       'Ohio': {2000: 1.5, 2001: 1.7, 2002: 3.6}}

In [40]:

frame3 = DataFrame(pop)frame3

Out[40]:

 NevadaOhio2000NaN1.520012.41.720022.93.6

In [41]:

frame3.T # 转置，行和列互换

Out[41]:

 200020012002NevadaNaN2.42.9Ohio1.51.73.6

In [42]:

DataFrame(pop, index=[2001, 2002, 2003])

Out[42]:

 NevadaOhio20012.41.720022.93.62003NaNNaN

In [43]:

pdata = {'Ohio': frame3['Ohio'][:-1],         'Nevada': frame3['Nevada'][:2]}DataFrame(pdata)

Out[43]:

 NevadaOhio2000NaN1.520012.41.7

In [44]:

frame3.index.name = 'year'; frame3.columns.name = 'state'frame3

Out[44]:

stateNevadaOhioyear  2000NaN1.520012.41.720022.93.6

In [45]:

frame3.values # DF返回二维数组

Out[45]:

array([[ nan,  1.5],       [ 2.4,  1.7],       [ 2.9,  3.6]])

In [46]:

frame2.values

Out[46]:

array([[2000L, 'Ohio', 1.5, nan],       [2001L, 'Ohio', 1.7, -1.2],       [2002L, 'Ohio', 3.6, nan],       [2001L, 'Nevada', 2.4, -1.5],       [2002L, 'Nevada', 2.9, -1.7]], dtype=object)

索引对象

In [47]:

obj = Series(range(3), index=['a', 'b', 'c'])index = obj.indexindex

Out[47]:

Index([u'a', u'b', u'c'], dtype='object')

In [48]:

index[1:]

Out[48]:

Index([u'b', u'c'], dtype='object')

In [49]:

index[1] = 'd' #索引对象不支持更改

TypeErrorTraceback (most recent call last)<ipython-input-49-676fdeb26a68> in <module>()----> 1 index[1] = 'd'D:\python2713\lib\anaconda_install\lib\site-packages\pandas\core\indexes\base.pyc in __setitem__(self, key, value)   1618    1619     def __setitem__(self, key, value):-> 1620         raise TypeError("Index does not support mutable operations")   1621    1622     def __getitem__(self, key):TypeError: Index does not support mutable operations

In [51]:

index = pd.Index(np.arange(3))index

Out[51]:

Int64Index([0, 1, 2], dtype='int64')

In [52]:

obj2 = Series([1.5, -2.5, 0], index=index)obj2

Out[52]:

0    1.51   -2.52    0.0dtype: float64

In [53]:

obj2.index is index

Out[53]:

True

In [54]:

frame3

Out[54]:

stateNevadaOhioyear  2000NaN1.520012.41.720022.93.6

In [55]:

'Ohio' in frame3.columns

Out[55]:

True

In [56]:

2003 in frame3.index

Out[56]:

False

基本功能

重建索引

In [58]:

obj = Series([4.5, 7.2, -5.3, 3.6], index=['d', 'b', 'a', 'c'])obj

Out[58]:

d    4.5b    7.2a   -5.3c    3.6dtype: float64

In [59]:

obj2 = obj.reindex(['a', 'b', 'c', 'd', 'e'])obj2

Out[59]:

a   -5.3b    7.2c    3.6d    4.5e    NaNdtype: float64

In [60]:

obj.reindex(['a', 'b', 'c', 'd', 'e'], fill_value=0) # 重新根据索引排序，有缺失值填入fill_value

Out[60]:

a   -5.3b    7.2c    3.6d    4.5e    0.0dtype: float64

In [61]:

obj3 = Series(['blue', 'purple', 'yellow'], index=[0, 2, 4])obj3.reindex(range(6), method='ffill') # 向前填充

Out[61]:

0      blue1      blue2    purple3    purple4    yellow5    yellowdtype: object

In [62]:

frame = DataFrame(np.arange(9).reshape((3, 3)), index=['a', 'c', 'd'],                  columns=['Ohio', 'Texas', 'California'])frame

Out[62]:

 OhioTexasCaliforniaa012c345d678

In [63]:

frame2 = frame.reindex(['a', 'b', 'c', 'd'])frame2

Out[63]:

 OhioTexasCaliforniaa0.01.02.0bNaNNaNNaNc3.04.05.0d6.07.08.0

In [64]:

states = ['Texas', 'Utah', 'California']frame.reindex(columns=states)

Out[64]:

 TexasUtahCaliforniaa1NaN2c4NaN5d7NaN8

In [66]:

frame.ix[['a', 'b', 'c', 'd'], states]

D:\python2713\lib\anaconda_install\lib\site-packages\ipykernel_launcher.py:1: DeprecationWarning: .ix is deprecated. Please use.loc for label based indexing or.iloc for positional indexingSee the documentation here:http://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate_ix  """Entry point for launching an IPython kernel.

Out[66]:

 TexasUtahCaliforniaa1.0NaN2.0bNaNNaNNaNc4.0NaN5.0d7.0NaN8.0

丢弃指定轴上的项

In [67]:

obj = Series(np.arange(5.), index=['a', 'b', 'c', 'd', 'e'])new_obj = obj.drop('c')new_obj

Out[67]:

a    0.0b    1.0d    3.0e    4.0dtype: float64

In [68]:

obj.drop(['d', 'c'])

Out[68]:

a    0.0b    1.0e    4.0dtype: float64

In [70]:

data = DataFrame(np.arange(16).reshape((4, 4)),                 index=['Ohio', 'Colorado', 'Utah', 'New York'],                 columns=['one', 'two', 'three', 'four'])

In [71]:

data

Out[71]:

 onetwothreefourOhio0123Colorado4567Utah891011New York12131415

In [72]:

data.drop(['Colorado', 'Ohio'])

Out[72]:

 onetwothreefourUtah891011New York12131415

In [73]:

data.drop('two', axis=1)

Out[73]:

 onethreefourOhio023Colorado467Utah81011New York121415

In [74]:

data.drop(['two', 'four'], axis=1)

Out[74]:

 onethreeOhio02Colorado46Utah810New York1214

索引、选取、过滤

In [77]:

obj = Series(np.arange(4.), index=['a', 'b', 'c', 'd'])obj

Out[77]:

a    0.0b    1.0c    2.0d    3.0dtype: float64

In [78]:

obj['b']

Out[78]:

1.0

In [79]:

obj[1]

Out[79]:

1.0

In [80]:

obj[2:4]

Out[80]:

c    2.0d    3.0dtype: float64

In [81]:

obj[['b', 'a', 'd']]

Out[81]:

b    1.0a    0.0d    3.0dtype: float64

In [82]:

obj[[1, 3]]

Out[82]:

b    1.0d    3.0dtype: float64

In [83]:

obj[obj < 2]

Out[83]:

a    0.0b    1.0dtype: float64

In [84]:

obj['b':'c']

Out[84]:

b    1.0c    2.0dtype: float64

In [85]:

obj['b':'c'] = 5obj

Out[85]:

a    0.0b    5.0c    5.0d    3.0dtype: float64

In [86]:

data = DataFrame(np.arange(16).reshape((4, 4)),                 index=['Ohio', 'Colorado', 'Utah', 'New York'],                 columns=['one', 'two', 'three', 'four'])data

Out[86]:

 onetwothreefourOhio0123Colorado4567Utah891011New York12131415

In [87]:

data['two']

Out[87]:

Ohio         1Colorado     5Utah         9New York    13Name: two, dtype: int32

In [88]:

data[['three', 'one']]

Out[88]:

 threeoneOhio20Colorado64Utah108New York1412

In [89]:

data[:2]

Out[89]:

 onetwothreefourOhio0123Colorado4567

In [90]:

data[data['three'] > 5]

Out[90]:

 onetwothreefourColorado4567Utah891011New York12131415

In [91]:

data < 5

Out[91]:

 onetwothreefourOhioTrueTrueTrueTrueColoradoTrueFalseFalseFalseUtahFalseFalseFalseFalseNew YorkFalseFalseFalseFalse

In [92]:

data[data < 5] = 0

In [93]:

data

Out[93]:

 onetwothreefourOhio0000Colorado0567Utah891011New York12131415

In [94]:

data.ix['Colorado', ['two', 'three']]

D:\python2713\lib\anaconda_install\lib\site-packages\ipykernel_launcher.py:1: DeprecationWarning: .ix is deprecated. Please use.loc for label based indexing or.iloc for positional indexingSee the documentation here:http://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate_ix  """Entry point for launching an IPython kernel.

Out[94]:

two      5three    6Name: Colorado, dtype: int32

In [95]:

data.ix[['Colorado', 'Utah'], [3, 0, 1]]

Out[95]:

 fouronetwoColorado705Utah1189

In [96]:

data.ix[2] # 选取单个列

Out[96]:

one       8two       9three    10four     11Name: Utah, dtype: int32

In [97]:

data.ix[:'Utah', 'two']

Out[97]:

Ohio        0Colorado    5Utah        9Name: two, dtype: int32

In [98]:

data.ix[:'Utah', 'two']

Out[98]:

Ohio        0Colorado    5Utah        9Name: two, dtype: int32

算数运算和数据对齐

In [99]:

s1 = Series([7.3, -2.5, 3.4, 1.5], index=['a', 'c', 'd', 'e'])s2 = Series([-2.1, 3.6, -1.5, 4, 3.1], index=['a', 'c', 'e', 'f', 'g'])

In [100]:

s1

Out[100]:

a    7.3c   -2.5d    3.4e    1.5dtype: float64

In [101]:

s2

Out[101]:

a   -2.1c    3.6e   -1.5f    4.0g    3.1dtype: float64

In [102]:

s1 + s2

Out[102]:

a    5.2c    1.1d    NaNe    0.0f    NaNg    NaNdtype: float64

In [103]:

df1 = DataFrame(np.arange(9.).reshape((3, 3)), columns=list('bcd'),                index=['Ohio', 'Texas', 'Colorado'])df2 = DataFrame(np.arange(12.).reshape((4, 3)), columns=list('bde'),                index=['Utah', 'Ohio', 'Texas', 'Oregon'])df1

Out[103]:

 bcdOhio0.01.02.0Texas3.04.05.0Colorado6.07.08.0

In [104]:

df2

Out[104]:

 bdeUtah0.01.02.0Ohio3.04.05.0Texas6.07.08.0Oregon9.010.011.0

In [105]:

df1 + df2 # 空值用NaN代替

Out[105]:

 bcdeColoradoNaNNaNNaNNaNOhio3.0NaN6.0NaNOregonNaNNaNNaNNaNTexas9.0NaN12.0NaNUtahNaNNaNNaNNaN

在算数方法中填充值

In [106]:

df1 = DataFrame(np.arange(12.).reshape((3, 4)), columns=list('abcd'))df2 = DataFrame(np.arange(20.).reshape((4, 5)), columns=list('abcde'))df1

Out[106]:

 abcd00.01.02.03.014.05.06.07.028.09.010.011.0

In [107]:

df2

Out[107]:

 abcde00.01.02.03.04.015.06.07.08.09.0210.011.012.013.014.0315.016.017.018.019.0

In [108]:

df1 + df2

Out[108]:

 abcde00.02.04.06.0NaN19.011.013.015.0NaN218.020.022.024.0NaN3NaNNaNNaNNaNNaN

In [109]:

df1.add(df2, fill_value=0)

Out[109]:

 abcde00.02.04.06.04.019.011.013.015.09.0218.020.022.024.014.0315.016.017.018.019.0

In [110]:

df1.reindex(columns=df2.columns, fill_value=0)

Out[110]:

 abcde00.01.02.03.0014.05.06.07.0028.09.010.011.00

DataFrame和Series间的运算

In [111]:

arr = np.arange(12.).reshape((3, 4))arr

Out[111]:

array([[  0.,   1.,   2.,   3.],       [  4.,   5.,   6.,   7.],       [  8.,   9.,  10.,  11.]])

In [112]:

arr[0]

Out[112]:

array([ 0.,  1.,  2.,  3.])

In [113]:

arr - arr[0]

Out[113]:

array([[ 0.,  0.,  0.,  0.],       [ 4.,  4.,  4.,  4.],       [ 8.,  8.,  8.,  8.]])

In [114]:

frame = DataFrame(np.arange(12.).reshape((4, 3)), columns=list('bde'),                  index=['Utah', 'Ohio', 'Texas', 'Oregon'])series = frame.ix[0]frame

D:\python2713\lib\anaconda_install\lib\site-packages\ipykernel_launcher.py:3: DeprecationWarning: .ix is deprecated. Please use.loc for label based indexing or.iloc for positional indexingSee the documentation here:http://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate_ix  This is separate from the ipykernel package so we can avoid doing imports until

Out[114]:

 bdeUtah0.01.02.0Ohio3.04.05.0Texas6.07.08.0Oregon9.010.011.0

In [115]:

series

Out[115]:

b    0.0d    1.0e    2.0Name: Utah, dtype: float64

In [116]:

frame - series

Out[116]:

 bdeUtah0.00.00.0Ohio3.03.03.0Texas6.06.06.0Oregon9.09.09.0

In [117]:

series2 = Series(range(3), index=['b', 'e', 'f'])frame + series2

Out[117]:

 bdefUtah0.0NaN3.0NaNOhio3.0NaN6.0NaNTexas6.0NaN9.0NaNOregon9.0NaN12.0NaN

In [118]:

series3 = frame['d']frame

Out[118]:

 bdeUtah0.01.02.0Ohio3.04.05.0Texas6.07.08.0Oregon9.010.011.0

In [119]:

series3

Out[119]:

Utah       1.0Ohio       4.0Texas      7.0Oregon    10.0Name: d, dtype: float64

In [120]:

frame.sub(series3, axis=0)

Out[120]:

 bdeUtah-1.00.01.0Ohio-1.00.01.0Texas-1.00.01.0Oregon-1.00.01.0

函数应用和映射

In [121]:

frame = DataFrame(np.random.randn(4, 3), columns=list('bde'),                  index=['Utah', 'Ohio', 'Texas', 'Oregon'])

In [122]:

frame

Out[122]:

 bdeUtah-0.2047080.478943-0.519439Ohio-0.5557301.9657811.393406Texas0.0929080.2817460.769023Oregon1.2464351.007189-1.296221

In [123]:

np.abs(frame) #求绝对值

Out[123]:

 bdeUtah0.2047080.4789430.519439Ohio0.5557301.9657811.393406Texas0.0929080.2817460.769023Oregon1.2464351.0071891.296221

In [124]:

f = lambda x: x.max() - x.min()

In [125]:

frame.apply(f)

Out[125]:

b    1.802165d    1.684034e    2.689627dtype: float64

In [126]:

frame.apply(f, axis=1)

Out[126]:

Utah      0.998382Ohio      2.521511Texas     0.676115Oregon    2.542656dtype: float64

In [127]:

def f(x):    return Series([x.min(), x.max()], index=['min', 'max'])frame.apply(f)

Out[127]:

 bdemin-0.5557300.281746-1.296221max1.2464351.9657811.393406

In [128]:

format = lambda x: '%.2f' % xframe.applymap(format)

Out[128]:

 bdeUtah-0.200.48-0.52Ohio-0.561.971.39Texas0.090.280.77Oregon1.251.01-1.30

In [129]:

frame['e'].map(format)

Out[129]:

Utah      -0.52Ohio       1.39Texas      0.77Oregon    -1.30Name: e, dtype: object

排序和排名

In [130]:

obj = Series(range(4), index=['d', 'a', 'b', 'c'])obj

Out[130]:

d    0a    1b    2c    3dtype: int64

In [131]:

obj.sort_index()

Out[131]:

a    1b    2c    3d    0dtype: int64

In [132]:

frame = DataFrame(np.arange(8).reshape((2, 4)), index=['three', 'one'],                  columns=['d', 'a', 'b', 'c'])frame.sort_index()

Out[132]:

 dabcone4567three0123

In [133]:

frame.sort_index(axis=1)

Out[133]:

 abcdthree1230one5674

In [134]:

frame.sort_index(axis=1, ascending=False)

Out[134]:

 dcbathree0321one4765

In [137]:

frame = DataFrame({'b': [4, 7, -3, 2], 'a': [0, 1, 0, 1]})frame

Out[137]:

 ab00411720-3312

In [138]:

frame.sort_index(by='b') #将b列按从小到大排序

D:\python2713\lib\anaconda_install\lib\site-packages\ipykernel_launcher.py:1: FutureWarning: by argument to sort_index is deprecated, pls use .sort_values(by=...)  """Entry point for launching an IPython kernel.

Out[138]:

 ab20-3312004117

In [139]:

frame.sort_index(by=['a', 'b']) # a,b列从小到大排列

D:\python2713\lib\anaconda_install\lib\site-packages\ipykernel_launcher.py:1: FutureWarning: by argument to sort_index is deprecated, pls use .sort_values(by=...)  """Entry point for launching an IPython kernel.

Out[139]:

 ab20-3004312117

In [140]:

obj = Series([7, -5, 7, 4, 2, 0, 4])obj.rank() # 排名

Out[140]:

0    6.51    1.02    6.53    4.54    3.05    2.06    4.5dtype: float64

In [141]:

obj.rank(method='first')# 出现的顺序进行排名

Out[141]:

0    6.01    1.02    7.03    4.04    3.05    2.06    5.0dtype: float64

In [142]:

obj.rank(ascending=False, method='max') #姜旭排名

Out[142]:

0    2.01    7.02    2.03    4.04    5.05    6.06    4.0dtype: float64

In [143]:

frame = DataFrame({'b': [4.3, 7, -3, 2], 'a': [0, 1, 0, 1],                   'c': [-2, 5, 8, -2.5]})frame

Out[143]:

 abc004.3-2.0117.05.020-3.08.0312.0-2.5

In [144]:

frame.rank(axis=1)

Out[144]:

 abc02.03.01.011.03.02.022.01.03.032.03.01.0

带有重复值得轴索引

In [145]:

obj = Series(range(5), index=['a', 'a', 'b', 'b', 'c'])obj

Out[145]:

a    0a    1b    2b    3c    4dtype: int64

In [146]:

obj.index.is_unique

Out[146]:

False

In [147]:

obj['a']

Out[147]:

a    0a    1dtype: int64

In [148]:

obj['c']

Out[148]:

4

In [149]:

df = DataFrame(np.random.randn(4, 3), index=['a', 'a', 'b', 'b'])df

Out[149]:

 012a0.2749920.2289131.352917a0.886429-2.001637-0.371843b1.669025-0.438570-0.539741b0.4769853.248944-1.021228

In [150]:

df.ix['b']

D:\python2713\lib\anaconda_install\lib\site-packages\ipykernel_launcher.py:1: DeprecationWarning: .ix is deprecated. Please use.loc for label based indexing or.iloc for positional indexingSee the documentation here:http://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate_ix  """Entry point for launching an IPython kernel.

Out[150]:

 012b1.669025-0.438570-0.539741b0.4769853.248944-1.021228

汇总和计算描述统计

In [151]:

df = DataFrame([[1.4, np.nan], [7.1, -4.5],                [np.nan, np.nan], [0.75, -1.3]],               index=['a', 'b', 'c', 'd'],               columns=['one', 'two'])df

Out[151]:

 onetwoa1.40NaNb7.10-4.5cNaNNaNd0.75-1.3

In [152]:

df.sum() # 求和（按列）

Out[152]:

one    9.25two   -5.80dtype: float64

In [153]:

df.sum(axis=1)# 求和（按行）

Out[153]:

a    1.40b    2.60c    0.00d   -0.55dtype: float64

In [154]:

df.mean(axis=1, skipna=False)# 求平均值（按行）

Out[154]:

a      NaNb    1.300c      NaNd   -0.275dtype: float64

In [155]:

df.idxmax() # 最大的值的标签

Out[155]:

one    btwo    ddtype: object

In [156]:

df.cumsum() # 累加和

Out[156]:

 onetwoa1.40NaNb8.50-4.5cNaNNaNd9.25-5.8

In [157]:

df.describe() #汇总多个统计数据

Out[157]:

 onetwocount3.0000002.000000mean3.083333-2.900000std3.4936852.262742min0.750000-4.50000025%1.075000-3.70000050%1.400000-2.90000075%4.250000-2.100000max7.100000-1.300000

In [158]:

obj = Series(['a', 'a', 'b', 'c'] * 4)obj.describe()

Out[158]:

count     16unique     3top        afreq       8dtype: object

In [ ]:

### 唯一值，估计值以及成员资格

In [160]:

obj = Series(['c', 'a', 'd', 'a', 'a', 'b', 'b', 'c', 'c'])

In [161]:

uniques = obj.unique()uniques

Out[161]:

array(['c', 'a', 'd', 'b'], dtype=object)

In [162]:

obj.value_counts()

Out[162]:

c    3a    3b    2d    1dtype: int64

In [163]:

pd.value_counts(obj.values, sort=False) #降频排列

Out[163]:

a    3c    3b    2d    1dtype: int64

In [164]:

mask = obj.isin(['b', 'c']) #判断是否包含mask

Out[164]:

0     True1    False2    False3    False4    False5     True6     True7     True8     Truedtype: bool

In [165]:

obj[mask]

Out[165]:

0    c5    b6    b7    c8    cdtype: object

In [166]:

data = DataFrame({'Qu1': [1, 3, 4, 3, 4],                  'Qu2': [2, 3, 1, 2, 3],                  'Qu3': [1, 5, 2, 4, 4]})data

Out[166]:

 Qu1Qu2Qu301211335241233244434

In [167]:

result = data.apply(pd.value_counts).fillna(0)result

Out[167]:

 Qu1Qu2Qu311.01.01.020.02.01.032.02.00.042.00.02.050.00.01.0

处理缺失数据

In [168]:

string_data = Series(['aardvark', 'artichoke', np.nan, 'avocado'])string_data

Out[168]:

0     aardvark1    artichoke2          NaN3      avocadodtype: object

In [169]:

string_data.isnull()

Out[169]:

0    False1    False2     True3    Falsedtype: bool

In [170]:

string_data[0] = Nonestring_data.isnull()

Out[170]:

0     True1    False2     True3    Falsedtype: bool

过滤缺失的数据

In [171]:

from numpy import nan as NAdata = Series([1, NA, 3.5, NA, 7])data.dropna() #干掉缺失的数据

Out[171]:

0    1.02    3.54    7.0dtype: float64

In [172]:

data[data.notnull()]

Out[172]:

0    1.02    3.54    7.0dtype: float64

In [173]:

data = DataFrame([[1., 6.5, 3.], [1., NA, NA],                  [NA, NA, NA], [NA, 6.5, 3.]])cleaned = data.dropna() # 一行中只要有all就会被干掉data

Out[173]:

 01201.06.53.011.0NaNNaN2NaNNaNNaN3NaN6.53.0

In [174]:

cleaned

Out[174]:

 01201.06.53.0

In [175]:

data.dropna(how='all') # 只干掉全为na的行

Out[175]:

 01201.06.53.011.0NaNNaN3NaN6.53.0

In [176]:

data[4] = NAdata

Out[176]:

 012401.06.53.0NaN11.0NaNNaNNaN2NaNNaNNaNNaN3NaN6.53.0NaN

In [178]:

data.dropna(axis=1, how='all') # axis = 1,干掉全为NA的列

Out[178]:

 01201.06.53.011.0NaNNaN2NaNNaNNaN3NaN6.53.0

In [179]:

df = DataFrame(np.random.randn(7, 3))df.ix[:4, 1] = NA; df.ix[:2, 2] = NAdf

D:\python2713\lib\anaconda_install\lib\site-packages\ipykernel_launcher.py:2: DeprecationWarning: .ix is deprecated. Please use.loc for label based indexing or.iloc for positional indexingSee the documentation here:http://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate_ix  

Out[179]:

 0120-0.577087NaNNaN10.523772NaNNaN2-0.713544NaNNaN3-1.860761NaN0.5601454-1.265934NaN-1.06351250.332883-2.359419-0.1995436-1.541996-0.970736-1.307030

In [180]:

df.dropna(thresh=3) #留一部分观测数据

Out[180]:

 01250.332883-2.359419-0.1995436-1.541996-0.970736-1.307030

填充缺失数据

In [181]:

df.fillna(0) #用0来填充缺失的数据

Out[181]:

 0120-0.5770870.0000000.00000010.5237720.0000000.0000002-0.7135440.0000000.0000003-1.8607610.0000000.5601454-1.2659340.000000-1.06351250.332883-2.359419-0.1995436-1.541996-0.970736-1.307030

In [182]:

df.fillna({1: 0.5, 3: -1})

Out[182]:

 0120-0.5770870.500000NaN10.5237720.500000NaN2-0.7135440.500000NaN3-1.8607610.5000000.5601454-1.2659340.500000-1.06351250.332883-2.359419-0.1995436-1.541996-0.970736-1.307030

In [185]:

# 通常会返回新对象，但也可以对现有对象进行就地修改_ = df.fillna(0, inplace=True)df

Out[185]:

 01200.2863500.377984-0.75388710.3312861.3497420.06987720.2466740.0000001.00481231.3271950.000000-1.54910640.0221850.0000000.00000050.8625800.0000000.000000

In [187]:

df = DataFrame(np.random.randn(6, 3))df.ix[2:, 1] = NA; df.ix[4:, 2] = NAdf

D:\python2713\lib\anaconda_install\lib\site-packages\ipykernel_launcher.py:2: DeprecationWarning: .ix is deprecated. Please use.loc for label based indexing or.iloc for positional indexingSee the documentation here:http://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate_ix  

Out[187]:

 01200.6702160.852965-0.9558691-0.023493-2.304234-0.6524692-1.218302NaN1.07462330.723642NaN1.0015434-0.503087NaNNaN5-0.726213NaNNaN

In [186]:

df.fillna(method='ffill')

Out[186]:

 01200.2863500.377984-0.75388710.3312861.3497420.06987720.2466740.0000001.00481231.3271950.000000-1.54910640.0221850.0000000.00000050.8625800.0000000.000000

In [188]:

df.fillna(method='ffill', limit=2)

Out[188]:

 01200.6702160.852965-0.9558691-0.023493-2.304234-0.6524692-1.218302-2.3042341.07462330.723642-2.3042341.0015434-0.503087NaN1.0015435-0.726213NaN1.001543

In [189]:

data = Series([1., NA, 3.5, NA, 7])data.fillna(data.mean())

Out[189]:

0    1.0000001    3.8333332    3.5000003    3.8333334    7.000000dtype: float64

层次化索引

In [190]:

data = Series(np.random.randn(10),              index=[['a', 'a', 'a', 'b', 'b', 'b', 'c', 'c', 'd', 'd'],                     [1, 2, 3, 1, 2, 3, 1, 2, 2, 3]])data

Out[190]:

a  1   -1.157719   2    0.816707   3    0.433610b  1    1.010737   2    1.824875   3   -0.997518c  1    0.850591   2   -0.131578d  2    0.912414   3    0.188211dtype: float64

In [191]:

data.index

Out[191]:

MultiIndex(levels=[[u'a', u'b', u'c', u'd'], [1, 2, 3]],           labels=[[0, 0, 0, 1, 1, 1, 2, 2, 3, 3], [0, 1, 2, 0, 1, 2, 0, 1, 1, 2]])

In [192]:

data['b']

Out[192]:

1    1.0107372    1.8248753   -0.997518dtype: float64

In [193]:

data['b':'c']

Out[193]:

b  1    1.010737   2    1.824875   3   -0.997518c  1    0.850591   2   -0.131578dtype: float64

In [194]:

data.ix[['b', 'd']]

D:\python2713\lib\anaconda_install\lib\site-packages\ipykernel_launcher.py:1: DeprecationWarning: .ix is deprecated. Please use.loc for label based indexing or.iloc for positional indexingSee the documentation here:http://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate_ix  """Entry point for launching an IPython kernel.

Out[194]:

b  1    1.010737   2    1.824875   3   -0.997518d  2    0.912414   3    0.188211dtype: float64

In [195]:

data[:, 2]

Out[195]:

a    0.816707b    1.824875c   -0.131578d    0.912414dtype: float64

In [196]:

data.unstack()

Out[196]:

 123a-1.1577190.8167070.433610b1.0107371.824875-0.997518c0.850591-0.131578NaNdNaN0.9124140.188211

In [197]:

data.unstack().stack()

Out[197]:

a  1   -1.157719   2    0.816707   3    0.433610b  1    1.010737   2    1.824875   3   -0.997518c  1    0.850591   2   -0.131578d  2    0.912414   3    0.188211dtype: float64

In [198]:

frame = DataFrame(np.arange(12).reshape((4, 3)),                  index=[['a', 'a', 'b', 'b'], [1, 2, 1, 2]],                  columns=[['Ohio', 'Ohio', 'Colorado'],                           ['Green', 'Red', 'Green']])frame

Out[198]:

  OhioColorado  GreenRedGreena10122345b1678291011

In [199]:

frame.index.names = ['key1', 'key2']frame.columns.names = ['state', 'color']frame

Out[199]:

 stateOhioColorado colorGreenRedGreenkey1key2   a10122345b1678291011

In [200]:

frame['Ohio']

Out[200]:

 colorGreenRedkey1key2  a101234b1672910

重排分级顺序

In [202]:

frame.swaplevel('key1', 'key2')

Out[202]:

 stateOhioColorado colorGreenRedGreenkey2key1   1a0122a3451b6782b91011

In [203]:

frame.sortlevel(1)

D:\python2713\lib\anaconda_install\lib\site-packages\ipykernel_launcher.py:1: FutureWarning: sortlevel is deprecated, use sort_index(level= ...)  """Entry point for launching an IPython kernel.

Out[203]:

 stateOhioColorado colorGreenRedGreenkey1key2   a1012b1678a2345b291011

In [204]:

frame.swaplevel(0, 1).sortlevel(0)

D:\python2713\lib\anaconda_install\lib\site-packages\ipykernel_launcher.py:1: FutureWarning: sortlevel is deprecated, use sort_index(level= ...)  """Entry point for launching an IPython kernel.

Out[204]:

 stateOhioColorado colorGreenRedGreenkey2key1   1a012b6782a345b91011

根据级别汇总统计

In [205]:

frame.sum(level='key2')

Out[205]:

stateOhioColoradocolorGreenRedGreenkey2   168102121416

In [206]:

frame.sum(level='color', axis=1)

Out[206]:

 colorGreenRedkey1key2  a121284b114722010

使用DataFrame的列

In [207]:

frame = DataFrame({'a': range(7), 'b': range(7, 0, -1),                   'c': ['one', 'one', 'one', 'two', 'two', 'two', 'two'],                   'd': [0, 1, 2, 0, 1, 2, 3]})frame

Out[207]:

 abcd007one0116one1225one2334two0443two1552two2661two3

In [208]:

frame2 = frame.set_index(['c', 'd'])frame2

Out[208]:

  abcd  one007116225two034143252361

In [209]:

frame.set_index(['c', 'd'], drop=False)

Out[209]:

  abcdcd    one007one0116one1225one2two034two0143two1252two2361two3

In [210]:

frame2.reset_index()

Out[210]:

 cdab0one0071one1162one2253two0344two1435two2526two361

其他关于 pandas话题

整数索引

In [211]:

ser = Series(np.arange(3.))ser.iloc[-1]

Out[211]:

2.0

In [212]:

ser

Out[212]:

0    0.01    1.02    2.0dtype: float64

In [213]:

ser2 = Series(np.arange(3.), index=['a', 'b', 'c'])ser2[-1]

Out[213]:

2.0

In [214]:

ser.ix[:1]

D:\python2713\lib\anaconda_install\lib\site-packages\ipykernel_launcher.py:1: DeprecationWarning: .ix is deprecated. Please use.loc for label based indexing or.iloc for positional indexingSee the documentation here:http://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate_ix  """Entry point for launching an IPython kernel.

Out[214]:

0    0.01    1.0dtype: float64

In [215]:

ser3 = Series(range(3), index=[-5, 1, 3])ser3

Out[215]:

-5    0 1    1 3    2dtype: int64

In [216]:

ser3.iloc[2]

Out[216]:

2

In [219]:

frame = DataFrame(np.arange(6).reshape((3, 2)), index=[2, 0, 1])frame.iloc[0]

Out[219]:

0    01    1Name: 2, dtype: int32