python3
来源:互联网 发布:在线端口检测 编辑:程序博客网 时间:2024/06/05 01:17
python3 安装jieba:
pip3 install jieba
参考:https://github.com/fxsjy/jieba
实例:
得到标签和创建mydict
import requestsfrom pyquery import PyQuery as pqfrom db import MongoClientfrom config import MY_DICTdb = MongoClient()def get_label(url): r = requests.get(url) r.encoding = 'utf-8' # 通过r.encoding设置页面编码 doc = pq(r.text) table = doc.find('body > div.body-wrapper > div.content-wrapper > div > div.main-content > table:nth-child(154) > tr').items() id = 0 for tr in table: if tr.find('td'): # 去掉th ''' 页面table有问题,单独修改一下 db.getCollection('hero').update( // query { "id" : 2 }, // update { '$set' : {'hero_name' : '幻翎', 'hero_name_list' : ['幻翎', '洛'], "join_time" : "2017年4月18日"} }, // options { "multi" : false, // update only one document "upsert" : false // insert a new document, if no existing document match the query } ); ''' id += 1 hero_name = tr.find('td:nth-child(2)').text().strip() hero_name_list = [] hero_name_list.append(tr.find('td:nth-child(2)').text().strip()) hero_name_list.append(tr.find('td:nth-child(3)').text().strip()) join_time = tr.find('td:nth-child(6)').text().strip() msg = { 'id' : id, 'hero_name' : hero_name, 'hero_name_list' : hero_name_list, 'join_time' : join_time } db.save(msg)def make_mydict(): with open(MY_DICT, mode='w', encoding='utf-8') as f: for name in db.get_hero_name_list(): print(name, file=f) # 直接换行if __name__ == '__main__': # get_label('http://baike.baidu.com/item/英雄联盟/4615671#4') make_mydict() # 创建词典 print('ok...')
分词器
import jiebaimport jieba.analysefrom db import MongoClientfrom config import MY_DICTclass Tokenizer(object): def __init__(self): self._db = MongoClient() # 载入自己的词库 jieba.load_userdict(MY_DICT) def get_hero_list(self): hero_list = [] with open(MY_DICT, mode='r', encoding='utf-8') as f: for hero in f: hero_list.append(hero.strip()) return hero_list def participle(self): hero_list = self.get_hero_list() print('/'.join(hero_list)) for room in self._db.get_rooms(): # 分词 [默认精确] msg = jieba.lcut(room['r_name']) label_list = set([w for w in msg if w in hero_list]) # 去重复 self._db.set_label(query={'r_id' : room['r_id']}, data={'$set' : {'r_label' : list(label_list)}}) print(msg, label_list)if __name__ == '__main__': # 分词器 tokenizer = Tokenizer() tokenizer.participle()
db
import pymongofrom config import *class MongoClient(object): def __init__(self): self._client = pymongo.MongoClient(MONGO_URL) def get_rooms(self): db = self._client[MONGO_DB] for room in db[MONGO_TABLE].find(): # 去掉limit yield { 'r_id' : room['r_id'], 'r_name' : room['r_name'] } def set_label(self, **kwargs): self._client[MONGO_DB][MONGO_TABLE].\ update(kwargs['query'], kwargs['data'], upsert=False) def save(self, msg): try: self._client[MONGO_DB][MONGO_HERO_NAME].insert(msg) except Exception as e: print("e: ", e) def get_hero_name_list(self): for hero_name in self._client[MONGO_DB][MONGO_HERO_NAME].find(): for name in hero_name['hero_name_list']: yield name
问题:
1. 运行的文件名和import xxx 的包名重复
import jiebajieba.cut("我来到北京清华大学")
AttributeError: 'module' object has no attribute 'cut'
不要将运行的文件名命名为jieba.py,自己撸自己当然出错了
阅读全文
0 0
- python3
- Python3
- python3
- Python3
- Python3
- Python3
- python3
- python3
- python3
- Python3
- python3
- python3
- Python3
- Python3
- python3
- Python3
- python3笔记
- python3 多线程
- Windows和Linux查看和更改mysql连接池
- java int类型与char类型转换
- uboot下载配置事项,使用uboot下载裸机程序,并运行
- python程序:检查字符串是否是回文(1)
- Android 获取SHA1命令
- python3
- Android开发,GPS获取实时时间并转为北京时间,定位信息,海拔高度,并进行显示
- 05、vue.js 之过滤器与实时计算
- 利用KMP算法,求顺序表存储的两个字符串的最长公共子串
- Html.DevExpress().TokenBox 属性说明,智能下拉
- 根据rgb获取颜色16进制代码
- BZOJ4884: [Lydsy2017年5月月赛]太空猫
- find、grep,which、whereis
- springcloud微服务一:spring boot基础项目搭建及问题处理