利用python进入数据分析之MovieLens_1M数据分析

数据设置

In [26]:

import pandas as pdimport osencoding = 'latin1'upath = os.path.expanduser('ch02/movielens/users.dat')rpath = os.path.expanduser('ch02/movielens/ratings.dat')mpath = os.path.expanduser('ch02/movielens/movies.dat')unames = ['user_id', 'gender', 'age', 'occupation', 'zip']rnames = ['user_id', 'movie_id', 'rating', 'timestamp']mnames = ['movie_id', 'title', 'genres']users = pd.read_csv(upath, sep='::', header=None, names=unames, encoding=encoding)ratings = pd.read_csv(rpath, sep='::', header=None, names=rnames, encoding=encoding)movies = pd.read_csv(mpath, sep='::', header=None, names=mnames, encoding=encoding)

In [6]:

users[:5]

Out[6]:

 user_idgenderageoccupationzip01F1104806712M56167007223M25155511734M4570246045M252055455

In [7]:

ratings[:5]

Out[7]:

 user_idmovie_idratingtimestamp011193597830076011661397830210921914397830196831340849783002754123555978824291

In [8]:

movies[:5]

Out[8]:

 movie_idtitlegenres01Toy Story (1995)Animation|Children's|Comedy12Jumanji (1995)Adventure|Children's|Fantasy23Grumpier Old Men (1995)Comedy|Romance34Waiting to Exhale (1995)Comedy|Drama45Father of the Bride Part II (1995)Comedy

In [9]:

ratings

Out[9]:

 user_idmovie_idratingtimestamp011193597830076011661397830210921914397830196831340849783002754123555978824291511197397830226861128759783020397128045978300719815944978302268919194978301368101595597882426811193849783017521212398497830228113129184978302124141103559783017531512791497830218816126873978824268171201849783017771813105597830171319127974978302039201232139783022052117203978300760221127059783000552315275978824195241234039783001032514859788243512611097497830195327117214978300055281154549788241392917453978824268...............100017960402762495670458410001806040103639567154551000181604050849567049721000182604010414957717678100018360403735496097165410001846040279149567155691000185604027941956716438100018660405275956704219100018760402003195671629410001886040535496482873410001896040201059577167951000190604020114956716113100019160403751496482878210001926040201959567039771000193604054149567152881000194604010775964828799100019560401079295671564810001966040549495670474610001976040202039567152881000198604020213956716374100019960402022595671620710002006040202859567045191000201604010804957717322100020260401089495670499610002036040109039567155181000204604010911956716541100020560401094595670488710002066040562595670474610002076040109649567156481000208604010974956715569

1000209 rows × 4 columns

数据合并

In [10]:

data = pd.merge(pd.merge(ratings, users), movies)data

Out[10]:

 user_idmovie_idratingtimestampgenderageoccupationziptitlegenres0111935978300760F11048067One Flew Over the Cuckoo's Nest (1975)Drama1211935978298413M561670072One Flew Over the Cuckoo's Nest (1975)Drama21211934978220179M251232793One Flew Over the Cuckoo's Nest (1975)Drama31511934978199279M25722903One Flew Over the Cuckoo's Nest (1975)Drama41711935978158471M50195350One Flew Over the Cuckoo's Nest (1975)Drama51811934978156168F18395825One Flew Over the Cuckoo's Nest (1975)Drama61911935982730936M11048073One Flew Over the Cuckoo's Nest (1975)Drama72411935978136709F25710023One Flew Over the Cuckoo's Nest (1975)Drama82811933978125194F25114607One Flew Over the Cuckoo's Nest (1975)Drama93311935978557765M45355421One Flew Over the Cuckoo's Nest (1975)Drama103911935978043535M18461820One Flew Over the Cuckoo's Nest (1975)Drama114211933978038981M25824502One Flew Over the Cuckoo's Nest (1975)Drama124411934978018995M451798052One Flew Over the Cuckoo's Nest (1975)Drama134711934977978345M18494305One Flew Over the Cuckoo's Nest (1975)Drama144811934977975061M25492107One Flew Over the Cuckoo's Nest (1975)Drama154911934978813972M181277084One Flew Over the Cuckoo's Nest (1975)Drama165311935977946400M25096931One Flew Over the Cuckoo's Nest (1975)Drama175411935977944039M50156723One Flew Over the Cuckoo's Nest (1975)Drama185811935977933866M25230303One Flew Over the Cuckoo's Nest (1975)Drama195911934977934292F50155413One Flew Over the Cuckoo's Nest (1975)Drama206211934977968584F35398105One Flew Over the Cuckoo's Nest (1975)Drama218011934977786172M56149327One Flew Over the Cuckoo's Nest (1975)Drama228111935977785864F25060640One Flew Over the Cuckoo's Nest (1975)Drama238811935977694161F45102476One Flew Over the Cuckoo's Nest (1975)Drama248911935977683596F56985749One Flew Over the Cuckoo's Nest (1975)Drama259511935977626632M45098201One Flew Over the Cuckoo's Nest (1975)Drama269611933977621789F251678028One Flew Over the Cuckoo's Nest (1975)Drama279911932982791053F11019390One Flew Over the Cuckoo's Nest (1975)Drama28102119351040737607M351920871One Flew Over the Cuckoo's Nest (1975)Drama2910411932977546620M251200926One Flew Over the Cuckoo's Nest (1975)Drama.................................1000179493330843962757020M251594040Home Page (1999)Documentary10001804802221821014866656M56140601Juno and Paycock (1930)Drama1000181481223082962932391M181425301Detroit 9000 (1973)Action|Crime100018248746244962781918F25470808Condition Red (1995)Action|Drama|Thriller1000183505914344962484364M451622652Stranger, The (1994)Action1000184594714344957190428F451697215Stranger, The (1994)Action1000185507718683962417299M25220037Truce, The (1996)Drama|War1000186594418681957197520F181027606Truce, The (1996)Drama|War100018751054043962337582M50718977Brother Minister: The Assassination of Malcolm...Documentary100018851854044963402617F35444485Brother Minister: The Assassination of Malcolm...Documentary100018955324045959619841M251727408Brother Minister: The Assassination of Malcolm...Documentary100019055434043960127592M251797401Brother Minister: The Assassination of Malcolm...Documentary1000191522025433961546137M25791436Six Ways to Sunday (1997)Comedy1000192575425434958272316F18160640Six Ways to Sunday (1997)Comedy100019352275913961475931M181064050Tough and Deadly (1995)Action|Drama|Thriller100019457955911958145253M25192688Tough and Deadly (1995)Action|Drama|Thriller1000195531336565960920392M56055406Lured (1947)Crime1000196532824384960838075F25491740Outside Ozona (1998)Drama|Thriller1000197533433233960796159F561346140Chain of Fools (2000)Comedy|Crime100019853341271960795494F561346140Silence of the Palace, The (Saimt el Qusur) (1...Drama1000199533433825960796159F561346140Song of Freedom (1936)Drama1000200542018433960156505F11914850Slappy and the Stinkers (1998)Children's|Comedy100020154332863960240881F351745014Nemesis 2: Nebula (1995)Action|Sci-Fi|Thriller1000202549435304959816296F351794306Smoking/No Smoking (1993)Comedy1000203555621983959445515M45692103Modulations (1998)Documentary1000204594921985958846401M181747901Modulations (1998)Documentary1000205567527033976029116M351430030Broken Vessels (1998)Drama1000206578028451958153068M181792886White Boys (1999)Drama1000207585136075957756608F182055410One Little Indian (1973)Comedy|Drama|Western1000208593829094957273353M25135401Five Wives, Three Secretaries and Me (1998)Documentary

1000209 rows × 10 columns

In [11]:

data.ix[0]

D:\python2713\lib\anaconda_install\lib\site-packages\ipykernel_launcher.py:1: DeprecationWarning: .ix is deprecated. Please use.loc for label based indexing or.iloc for positional indexingSee the documentation here:http://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate_ix  """Entry point for launching an IPython kernel.

Out[11]:

user_id                                            1movie_id                                        1193rating                                             5timestamp                                  978300760gender                                             Fage                                                1occupation                                        10zip                                            48067title         One Flew Over the Cuckoo's Nest (1975)genres                                         DramaName: 0, dtype: object

计算电影平均分

In [34]:

import sysreload(sys)sys.setdefaultencoding('latin1')mean_ratings = data.pivot_table('rating', index='title',columns='gender', aggfunc='mean')

In [38]:

mean_ratings[:5]

Out[38]:

genderFMtitle  $1,000,000 Duck (1971)3.3750002.761905'Night Mother (1986)3.3888893.352941'Til There Was You (1997)2.6756762.733333'burbs, The (1989)2.7934782.962085...And Justice for All (1979)3.8285713.689024

In [39]:

ratings_by_title = data.groupby('title').size()  #对title进行分组

In [40]:

ratings_by_title[:5]

Out[40]:

title$1,000,000 Duck (1971)            37'Night Mother (1986)              70'Til There Was You (1997)         52'burbs, The (1989)               303...And Justice for All (1979)    199dtype: int64

In [41]:

active_titles = ratings_by_title.index[ratings_by_title >= 250] # 获得评论数据大于250的电影

In [42]:

active_titles[:10]

Out[42]:

Index([u''burbs, The (1989)', u'10 Things I Hate About You (1999)',       u'101 Dalmatians (1961)', u'101 Dalmatians (1996)',       u'12 Angry Men (1957)', u'13th Warrior, The (1999)',       u'2 Days in the Valley (1996)', u'20,000 Leagues Under the Sea (1954)',       u'2001: A Space Odyssey (1968)', u'2010 (1984)'],      dtype='object', name=u'title')

In [43]:

mean_ratings = mean_ratings.ix[active_titles]mean_ratings

Out[43]:

genderFMtitle  'burbs, The (1989)2.7934782.96208510 Things I Hate About You (1999)3.6465523.311966101 Dalmatians (1961)3.7914443.500000101 Dalmatians (1996)3.2400002.91121512 Angry Men (1957)4.1843974.32842113th Warrior, The (1999)3.1120003.1680002 Days in the Valley (1996)3.4888893.24481320,000 Leagues Under the Sea (1954)3.6701033.7092052001: A Space Odyssey (1968)3.8255814.1297382010 (1984)3.4468093.41371228 Days (2000)3.2094242.97770739 Steps, The (1935)3.9655174.10769254 (1998)2.7017542.7821787th Voyage of Sinbad, The (1958)3.4090913.6588798MM (1999)2.9062502.850962About Last Night... (1986)3.1886793.140909Absent Minded Professor, The (1961)3.4693883.446809Absolute Power (1997)3.4691363.327759Abyss, The (1989)3.6592363.689507Ace Ventura: Pet Detective (1994)3.0000003.197917Ace Ventura: When Nature Calls (1995)2.2696632.543333Addams Family Values (1993)3.0000002.878531Addams Family, The (1991)3.1861703.163498Adventures in Babysitting (1987)3.4557823.208122Adventures of Buckaroo Bonzai Across the 8th Dimension, The (1984)3.3085113.402321Adventures of Priscilla, Queen of the Desert, The (1994)3.9890713.688811Adventures of Robin Hood, The (1938)4.1666673.918367African Queen, The (1951)4.3242324.223822Age of Innocence, The (1993)3.8270683.339506Agnes of God (1985)3.5348843.244898.........White Men Can't Jump (1992)3.0287773.231061Who Framed Roger Rabbit? (1988)3.5693783.713251Who's Afraid of Virginia Woolf? (1966)4.0297034.096939Whole Nine Yards, The (2000)3.2965523.404814Wild Bunch, The (1969)3.6363644.128099Wild Things (1998)3.3920003.459082Wild Wild West (1999)2.2754492.131973William Shakespeare's Romeo and Juliet (1996)3.5326093.318644Willow (1988)3.6586833.453543Willy Wonka and the Chocolate Factory (1971)4.0639533.789474Witness (1985)4.1158543.941504Wizard of Oz, The (1939)4.3550304.203138Wolf (1994)3.0740742.899083Women on the Verge of a Nervous Breakdown (1988)3.9343073.865741Wonder Boys (2000)4.0437963.913649Working Girl (1988)3.6067423.312500World Is Not Enough, The (1999)3.3375003.388889Wrong Trousers, The (1993)4.5882354.478261Wyatt Earp (1994)3.1470593.283898X-Files: Fight the Future, The (1998)3.4894743.493797X-Men (2000)3.6823103.851702Year of Living Dangerously (1982)3.9512203.869403Yellow Submarine (1968)3.7142863.689286You've Got Mail (1998)3.5424243.275591Young Frankenstein (1974)4.2899634.239177Young Guns (1988)3.3717953.425620Young Guns II (1990)2.9347832.904025Young Sherlock Holmes (1985)3.5147063.363344Zero Effect (1998)3.8644073.723140eXistenZ (1999)3.0985923.289086

1216 rows × 2 columns

In [44]:

mean_ratings = mean_ratings.rename(index={'Seven Samurai (The Magnificent Seven) (Shichinin no samurai) (1954)':                           'Seven Samurai (Shichinin no samurai) (1954)'})

In [45]:

top_female_ratings = mean_ratings.sort_index(by='F', ascending=False)# 获取女性观众最喜欢的电影top_female_ratings[:10]

Out[45]:

genderFMtitle  Close Shave, A (1995)4.6444444.473795Wrong Trousers, The (1993)4.5882354.478261Sunset Blvd. (a.k.a. Sunset Boulevard) (1950)4.5726504.464589Wallace & Gromit: The Best of Aardman Animation (1996)4.5631074.385075Schindler's List (1993)4.5626024.491415Shawshank Redemption, The (1994)4.5390754.560625Grand Day Out, A (1992)4.5378794.293255To Kill a Mockingbird (1962)4.5366674.372611Creature Comforts (1990)4.5138894.272277Usual Suspects, The (1995)4.5133174.518248