redis去重

import hashlibimport redis"""利用redis的集合不允许添加重复元素来进行去重"""import configparserconfig = configparser.ConfigParser()config.read('db_conf')host = config['redis']['host']port = config.getint('redis', 'port')db = config.getint('redis', 'db')password = config.get('redis', 'password')def redis_init():    """    初始化redis    :param parasecname:    :return: redis连接池    """    pool = redis.ConnectionPool(host=host, port=port, db=db, password=password)    # r = redis.Redis(connection_pool=pool)    r = redis.StrictRedis(connection_pool=pool)    return pool, rdef sha1(x):    sha1obj = hashlib.sha1()    sha1obj.update(x.encode('utf-8'))    hash_value = sha1obj.hexdigest()    return hash_valuedef rem_redis(r, check_str, set_name):    hash_value = sha1(check_str)    r.srem(set_name, hash_value)def add_redis(r, check_str, set_name):    """    将check_str加入redis连接r的set_name集合中    :param r:redis连接    :param check_str:被添加的字符串    :param set_name:项目所使用的集合名称，建议如下格式：”projectname:task_remove_repeate“    :return:    """    hash_value = sha1(check_str)    r.sadd(set_name, hash_value)def is_new(r, check_str, set_name):    """    check_str是否为set_name的新元素 新元素返回1 重复元素返回0    :param r:    :param check_str:    :param set_name:    :return:    """    hash_value = sha1(check_str)#加密，节省空间    return 0 if r.sismember(set_name, hash_value) else 1def redis_close(pool):    """    释放redis连接池    :param pool:    :return:    """    pool.disconnect()