简单分析cloudkitty的hashmap计费代码
来源：互联网 发布：10大网络恐怖推理图 编辑：程序博客网 时间：2024/06/12 07:12
1. cloudkitty的计费算法
本文代码主要基于cloudkitty的ocata版本
1.1. 服务启动
在代码cloudkitty/cli/processor.py中的
def main(): service.prepare_service() processor = orchestrator.Orchestrator() try: processor.process() except KeyboardInterrupt: processor.terminate()if __name__ == '__main__': main()
1.2. 周期性执行计费框架orchestrator执行process（）
def process(self): while True:#这里没有做任何事，应该是预留 self.process_messages()#获取包含cloudkitty的并含有ratting角色的项目 self._load_tenant_list() while len(self._tenants): for tenant in self._tenants[:]:#通过tooz获取项目锁，防止冲突 lock = self._lock(tenant) if lock.acquire(blocking=False): if not self._check_state(tenant): self._tenants.remove(tenant) else:#处理计费统计任务 worker = Worker(self.collector, self.storage, tenant) worker.run() lock.release() self.coord.heartbeat() # NOTE(sheeprine): Slow down looping if all tenants are # being processed eventlet.sleep(1) # FIXME(sheeprine): We may cause a drift here eventlet.sleep(self._period)
1.3. _check_status说明
def _check_state(self, tenant_id): timestamp = self.storage.get_state(tenant_id) return ck_utils.check_time_state(timestamp, self._period, self._wait_time)get_status位于cloudkitty/storage/sqlalchemy/__init__.py中def get_state(self, tenant_id=None): session = db.get_session()#到数据库表rated_data_frames获取指定租户向的计费数据，并且根据时间记录的开始时间降序排列 q = utils.model_query( self.frame_model, session) if tenant_id: q = q.filter( self.frame_model.tenant_id == tenant_id) q = q.order_by( self.frame_model.begin.desc())#获取取到的第一个数据，如果初始情况下，可能获取不到数据 r = q.first() if r: return ck_utils.dt2ts(r.begin)check_time_status位于cloudkitty/utils.py中def check_time_state(timestamp=None, period=0, wait_time=0):#获取不到时间，直接获取每个月的起始时间 if not timestamp: month_start = get_month_start() return dt2ts(month_start)#获取到时间了，就看上次计费时间+统计周期是否已经到达预定值，如果达到预定值，则开始计费 now = utcnow_ts() next_timestamp = timestamp + period if next_timestamp + wait_time < now: return next_timestamp return 0
1.4. Worker run说明
该代码位于cloudkitty/orchestrator.py中
def run(self): while True:#判读是否到计费周期了，timestamp为起始周期，一般获取到的都是当前第一个计费周期 timestamp = self.check_state() if not timestamp: break #服务包括：compute,image,volume,network.bw.in,network.bw.out,network.floating for service in CONF.collect.services: try: try:#根据stevestore的插件机制，获取指定服务在指定时间段的事件信息 data = self._collect(service, timestamp) except collector.NoDataCollected: raise except Exception as e: LOG.warning( _LW('Error while collecting service ' '%(service)s: %(error)s'), {'service': service, 'error': e}) raise collector.NoDataCollected('', service) except collector.NoDataCollected: begin = timestamp end = begin + self._period for processor in self._processors: processor.obj.nodata(begin, end) self._storage.nodata(begin, end, self._tenant_id) else: # Rating# 根据stevestore的插件功能，查询’cloudkitty.rating.processors’提供的计费策略，并进行计费，当前主要讲hashmap的计费代码 for processor in self._processors: processor.obj.process(data) # Writing self._storage.append(data, self._tenant_id) # We're getting a full period so we directly commit self._storage.commit(self._tenant_id)# _collect是根据service的起始时间后端存储中获取对应的数据def _collect(self, service, start_timestamp): next_timestamp = start_timestamp + self._period raw_data = self._collector.retrieve(service, start_timestamp, next_timestamp, self._tenant_id)# raw_data的格式为：instance = {'instance_id':xx, 'flavor': xxx}raw_data = {'compute': [{'desc': instance, 'vol': {'unit': 'instance', 'qty': 1.0}}]} if raw_data: return [{'period': {'begin': start_timestamp, 'end': next_timestamp}, 'usage': raw_data}]# retrieve (本文以ceilometer， compute为例)，其中ceilometer的retrieve定义在基类中，最终走到ceilometer的get_compute方法中代码路径：cloudkitty/collector/ceilometer.pydef get_compute(self, start, end=None, project_id=None, q_filter=None):#这个方法是调用ceilometer的statistics的统计方法，统计所有云主机在指定时间的cpu统计值，在这段时间外创建/删除的云主机则没有统计值信息 active_instance_ids = self.active_resources('cpu', start, end, project_id, q_filter) compute_data = [] for instance_id in active_instance_ids: if not self._cacher.has_resource_detail('compute', instance_id):#调用ceilometer resource-show获取指定资源的信息 raw_resource = self._conn.resources.get(instance_id)#组装instance的信息 instance = self.t_ceilometer.strip_resource_data('compute', raw_resource) self._cacher.add_resource_detail('compute', instance_id, instance) instance = self._cacher.get_resource_detail('compute', instance_id)# 将数据转化成以下格式data = {'desc': instance, 'vol': {'unit': 'instance', 'qty': 1.0}} compute_data.append( self.t_cloudkitty.format_item(instance, self.units_mappings[ "compute"], 1)) if not compute_data: raise collector.NoDataCollected(self.collector_name, 'compute')# 将数据转化成以下格式data = {'compute': [{'desc': instance, 'vol': {'unit': 'instance', 'qty': 1.0}}]} return self.t_cloudkitty.format_service('compute', compute_data)#strip_resource_data就是根据从ceilometer中获取的资源，组装数据简体def strip_resource_data(self, res_type, res_data): res_type = res_type.replace('.', '_') strip_func = getattr(self, '_strip_' + res_type, None) if strip_func: return strip_func(res_data) return self.generic_strip(res_type, res_data) or res_data#最终在cloudkitty/transformer/ceilometer.py 查询到_strip_computer的函数def _strip_compute(self, data): res_data = self.generic_strip('compute', data) res_data['instance_id'] = data.resource_id res_data['project_id'] = data.project_id res_data['user_id'] = data.user_id res_data['metadata'] = {} for field in data.metadata: if field.startswith('user_metadata'): res_data['metadata'][field[14:]] = data.metadata[field] return res_data# generic_strip就是到对应的compute_map获取指定key值的数据，compute_map格式如下compute_map = { 'name': ['display_name'], 'flavor': ['flavor.name', 'instance_type'], 'vcpus': ['vcpus'], 'memory': ['memory_mb'], 'image_id': ['image.id', 'image_meta.base_image_ref'], 'availability_zone': [ 'availability_zone', 'OS-EXT-AZ.availability_zone'],}1.5. hashmap计费
process的计费代码该代码位于cloudkitty/rating/hash/__init__.py中
def process(self, data):#data的格式为instance = {'instance_id':xx, 'flavor': xxx}raw_data = {'compute': [{'desc': instance, 'vol': {'unit': 'instance', 'qty': 1.0}}]}data= [{'period': {'begin': start_timestamp,'end': next_timestamp}, 'usage': raw_data}] for cur_data in data: cur_usage = cur_data['usage'] for service_name, service_data in cur_usage.items(): for item in service_data: self._res = {} self.process_services(service_name, item) self.process_fields(service_name, item) self.add_rating_informations(item) return data1.6. process_services说明
process_services的计费代码该代码位于cloudkitty/rating/hash/__init__.py中
def process_services(self, service_name, data):#self._entries策略的加载见，”加载hashmap计费策略”“最终的格式如下：{ 'volume': { 'thresholds': { u'volume_thresholds': { Decimal('1.00000000'): { 'cost': Decimal('5.00000000'), 'type': u'flat' } } }, 'mappings': {} }, 'compute': { 'fields': { u'flavor': { 'thresholds': {}, 'mappings': { u'instance_flavor': { u'm1.tiny': { 'cost': Decimal('20.00000000'), 'type': u'flat' } } } } }, 'thresholds': { u'instance_flavor': { Decimal('2.00000000'): { 'cost': Decimal('2.00000000'), 'type': u'flat' } } }, 'mappings': { u'instance_flavor': { 'cost': Decimal('20.00000000'), 'type': u'flat' } } }}”if service_name not in self._entries: return service_mappings = self._entries[service_name]['mappings']"""service_mappings = {'instance_flavor': {'cost': Decimal('20.00000000'),'type': u'flat'}""" for group_name, mapping in service_mappings.items():#刷新self._res的值self.res = {"instance_flavor": {'flat': 20, 'rate': 1, 'threshold': { 'level': -1, 'cost': 0, 'type': 'flat', 'scope': 'field'}}} self.update_result(group_name, mapping['type'], mapping['cost']) service_thresholds = self._entries[service_name]['thresholds']"""service_thresholds = {'instance_flavor': {Decimal('2.00000000'): {'cost': Decimal('2.00000000'),'type': u'flat'}}原始数据 qty的值为 1.0 """ self.process_thresholds(service_thresholds, data['vol']['qty'], 'service')#self.update_result说明, 这个时间is_threshold为false，该函数主要在刷新资源计算值def update_result(self, group, map_type, cost, level=0, is_threshold=False, threshold_scope='field'): if group not in self._res: self._res[group] = {'flat': 0, 'rate': 1, 'threshold': { 'level': -1, 'cost': 0, 'type': 'flat', 'scope': 'field'}} if is_threshold: best = self._res[group]['threshold']['level'] if level > best: self._res[group]['threshold']['level'] = level self._res[group]['threshold']['cost'] = cost self._res[group]['threshold']['type'] = map_type self._res[group]['threshold']['scope'] = threshold_scope else: if map_type == 'rate': self._res[group]['rate'] *= cost elif map_type == 'flat': new_flat = cost cur_flat = self._res[group]['flat'] if new_flat > cur_flat: self._res[group]['flat'] = new_flat1.7. Process_thrssholds说明
def process_thresholds(self,threshold_groups, cmp_level, threshold_type): #service_thresholds = {'instance_flavor': {Decimal('2.00000000'): {'cost': Decimal('2.00000000'),'type': u'flat'}}# cmp_level 假设为 1.0# threshold_type 为servicefor group_name, thresholds in threshold_groups.items(): for threshold_level, threshold in thresholds.items(): if cmp_level >= threshold_level: self.update_result( group_name, threshold['type'], threshold['cost'], threshold_level, True, threshold_type)#刷新self._res的值self.res = {"instance_flavor": {'flat': 20, 'rate': 1, 'threshold': { 'level': 2, 'cost': 2, 'type': 'flat', 'scope': 'field'}}}1.8. process_fields说明
def process_fields(self, service_name, data): if service_name not in self._entries: return if 'fields' not in self._entries[service_name]: return desc_data = data['desc'] field_mappings = self._entries[service_name]['fields']#field_mappings = { 'flavor': {'thresholds': {}, 'mappings': {'instance_flavor': {'m1.tiny': {'cost': Decimal('20.00000000'),'type': u'flat'}}} } for field_name, group_mappings in field_mappings.items(): if field_name not in desc_data: continue cmp_value = desc_data[field_name]#这个只是简单的根据flavor的值，修改下self._res中的cost值 self.process_mappings(group_mappings['mappings'], cmp_value) if group_mappings['thresholds']: self.process_thresholds(group_mappings['thresholds'], decimal.Decimal(cmp_value), 'field')#process_mapping的流程def process_mappings(self, mapping_groups, cmp_value): for group_name, mappings in mapping_groups.items(): for mapping_value, mapping in mappings.items(): if cmp_value == mapping_value: self.update_result( group_name, mapping['type'], mapping['cost'])1.9. add_rating_informations说明
def add_rating_informations(self, data):self.res = {"instance_flavor": {'flat': 20, 'rate': 1, 'threshold': { 'level': 2, 'cost': 2, 'type': 'flat', 'scope': 'field'}}} if 'rating' not in data: data['rating'] = {'price': 0}#最终的计费规则则是再最外层的flat的基础价格上加上threshold的价格数据#并将该数据保存到数据库中，实现周期计费的功能 for entry in self._res.values(): rate = entry['rate'] flat = entry['flat'] if entry['threshold']['scope'] == 'field': if entry['threshold']['type'] == 'flat': flat += entry['threshold']['cost'] else: rate *= entry['threshold']['cost'] res = rate * flat # FIXME(sheeprine): Added here to ensure that qty is decimal res *= decimal.Decimal(data['vol']['qty']) if entry['threshold']['scope'] == 'service': if entry['threshold']['type'] == 'flat': res += entry['threshold']['cost'] else: res *= entry['threshold']['cost'] data['rating']['price'] += res2. 加载hashmap计费策略_load_rates
改代码位于cloudkitty/rating/hash/__init__.py中
class HashMap(rating.RatingProcessorBase): """HashMap rating module. HashMap can be used to map arbitrary fields of a resource to different costs. """ module_name = 'hashmap' description = 'HashMap rating module.' hot_config = True config_controller = root_api.HashMapConfigController db_api = hash_db_api.get_instance() def __init__(self, tenant_id=None): super(HashMap, self).__init__(tenant_id) self._entries = {} self._res = {}#加载计费策略 self._load_rates()在计费算中的process.process_services代码中，会用到self._entries的全局变量，这个是计费策略，加载计费策略的代码如下：
def _load_rates(self): self._entries = {} hashmap = hash_db_api.get_instance()# 从hashmap_services的表中获取当前的service列表 
services_uuid_list = hashmap.list_services() for service_uuid in services_uuid_list: service_db = hashmap.get_service(uuid=service_uuid) service_name = service_db.name#加载mappings与thresholds的配置 self._load_service_entries(service_name, service_uuid) fields_uuid_list = hashmap.list_fields(service_uuid) 
for field_uuid in fields_uuid_list: field_db = hashmap.get_field(uuid=field_uuid) field_name = field_db.name self._load_field_entries(service_name, field_name, field_uuid)#经过上面转化，最终的self._entries格式为：{ 'volume': { 'thresholds': { u'volume_thresholds': { Decimal('1.00000000'): { 'cost': Decimal('5.00000000'), 'type': u'flat' } } }, 'mappings': {} }, 'compute': { 'fields': { u'flavor': { 'thresholds': {}, 'mappings': { u'instance_flavor': { u'm1.tiny': { 'cost': Decimal('20.00000000'), 'type': u'flat' } } } } }, 'thresholds': { u'instance_flavor': { Decimal('2.00000000'): { 'cost': Decimal('2.00000000'), 'type': u'flat' } } }, 'mappings': { u'instance_flavor': { 'cost': Decimal('20.00000000'), 'type': u'flat' } } }}2.1. _load_service_entries代码分析
#代码_load_service_entries()的逻辑，加载mappings与thresholds的配置def _load_service_entries(self, service_name, service_uuid): self._entries[service_name] = dict() for entry_type in ('mappings', 'thresholds'): for tenant in (None, self._tenant_id): self._update_entries(entry_type, self._entries[service_name], service_uuid=service_uuid, tenant_uuid=tenant) #代码最终的entries的结构如下： #界面创建的hashmap数据见hashmap截图#其实compute中的service mappings 与 service thresholds可以不设置的#由于本文为了说明其算法，简单的进行了设置# 并且以下的 mappings中的"m1.tiny" :{"type":"flat", "cost":20},实际上是没有的,这个是在关联filed的时候，才会获取到的mappings = { "instance_flavor": { "m1.tiny" :{"type":"flat", "cost":20}, # 有value "type" :"flat", "cost":20 } "_DEFAULT_": {} } }thresholds = { "instance_flavor": { "threshold_level":{"type": "flat", "cost": 5}, "2":{"type": "flat", "cost": 2} }, "_DEFAULT_": {} }self._entries = { "compute": { "thresholds":thresholds, "mappings": mappings }} #代码_update_entries的逻辑def _update_entries(self, entry_type, root, service_uuid=None, field_uuid=None, tenant_uuid=None): hashmap = hash_db_api.get_instance()#分别调用list_mappings, 以及 list_threshoulds方法，获取相关计费数据 
list_func = getattr(hashmap, 'list_{}'.format(entry_type)) entries_uuid_list = list_func( service_uuid=service_uuid, field_uuid=field_uuid, tenant_uuid=tenant_uuid)#分别调用_load_mappings, 以及 _load_threshoulds方法，组装计费数据 load_func = getattr(self, '_load_{}'.format(entry_type)) entries = load_func(entries_uuid_list)#将值保持到self._entries[service_name]中 if entry_type in root: res = root[entry_type] for group, values in entries.items(): if group in res: res[group].update(values) else: res[group] = values else: root[entry_type] = entries#_load_mappings的逻辑def _load_mappings(self, mappings_uuid_list):#加载后，返回的数据格式如下"""mappings = { "instance_flavor": { "m1.tiny" :{"type":"flat", "cost":20}, # 有value "type" :"flat", # 没有value "cost":20 # 没有value },"_DEFAULT_": {} }}""" 
hashmap = hash_db_api.get_instance() mappings = {} for mapping_uuid in mappings_uuid_list: mapping_db = hashmap.get_mapping(uuid=mapping_uuid) if mapping_db.group_id: group_name = mapping_db.group.name else: group_name = '_DEFAULT_' if group_name not in mappings: mappings[group_name] = {} current_scope = mappings[group_name] mapping_value = mapping_db.value#含有value的情况是在flavor创建该flavor的价格的时候才会输入的，数据库表中没有对应的service_id#在创建的最外层创建的时候，数据库表中有关联到具体的service_id if mapping_value: current_scope[mapping_value] = {} current_scope = current_scope[mapping_value] current_scope['type'] = mapping_db.map_type current_scope['cost'] = mapping_db.cost return mappings#_load_thresholds的逻辑def _load_thresholds(self, thresholds_uuid_list):#加载后，返回的数据格式如下"""thresholds = { "instance_flavor": { "threshold_level":{"type": "flat", "cost": 5}, "2":{"type": "flat", "cost": 2} }, "_DEFAULT_": {}}""" 
hashmap = hash_db_api.get_instance() thresholds = {} for threshold_uuid in thresholds_uuid_list: threshold_db = hashmap.get_threshold(uuid=threshold_uuid) if threshold_db.group_id: group_name = threshold_db.group.name else: group_name = '_DEFAULT_' if group_name not in thresholds: thresholds[group_name] = {} current_scope = thresholds[group_name] threshold_level = threshold_db.level current_scope[threshold_level] = {} current_scope = current_scope[threshold_level] current_scope['type'] = threshold_db.map_type current_scope['cost'] = threshold_db.cost return thresholds2.2. _load_field_entries代码分析
def _load_field_entries(self, service_name, field_name, field_uuid):#参数例子：compute, flavor, 0b250b5d-78f5-4bb3-835f-2f9b5be67b05 if service_name not in self._entries: self._entries[service_name] = {} if 'fields' not in self._entries[service_name]: self._entries[service_name]['fields'] = {} #self._entries变成 { "compute": { "thresholds":thresholds, "mappings": mappings, " fields ": {“flavor”: "m1.tiny" :{"type":"flat", "cost":20},}, }}scope = self._entries[service_name]['fields'][field_name] = {} for entry_type in ('mappings', 'thresholds'): for tenant in (None, self._tenant_id): self._update_entries( entry_type, scope, field_uuid=field_uuid, tenant_uuid=tenant)阅读全文
0 0
- 简单分析cloudkitty的hashmap计费代码
- OpenStack计费服务Cloudkitty分析(一)
- OpenStack计费服务Cloudkitty分析 计费核心(二)
- OpenStack计费项目CloudKitty的强化及运用
- OpenStack计费组件 - Cloudkitty介绍
- OpenStack计费项目Cloudkitty系列详解
- 简单的计费系统
- HashMap的简单源码分析
- HashMap的使用，简单小代码
- HashMap的实现原理简单分析
- 手机卡计费方式的分析与选择
- 北京出租车计费的信息与分析
- HashMap简单分析
- HashMap简单原理分析
- HashMap实现原理简单分析
- HashMap、HashTable、HashSet分析比较以及TreeSet的简单介绍
- 简单分析HashMap及其线程安全的Map类
- 汇编代码的简单分析
- BZOJ 3230 相似子串 后缀数组+二分+ST表
- Mybatis的一级缓存和二级缓存
- openwrt profile增加默认包及默认宏
- 自动化测试工具monkey
- View的scrollTo/scrollBy
- 简单分析cloudkitty的hashmap计费代码
- Oracle学习笔记day05——DDL语言
- 剑指offer:左旋转字符串
- String.format()的使用
- 2017多校联合第七场1010/hdu 6129 Just do it（递推／杨辉三角）
- effective C++ 条款三 解读
- org.apache.jasper.JasperException: Unable to compile class for JSP
- XML布局和代码快中设置前景/背景图片
- netifd支持的设备类型(netifd-2017-03-31)
