五、caffe 之gflags&glogs解析
来源:互联网 发布:淘宝引爆流量 编辑:程序博客网 时间:2024/05/23 12:53
为了弄懂是caffe/tool/caffe.cpp的源文件代码,首先提供一下原代码;
#ifdef WITH_PYTHON_LAYER#include "boost/python.hpp"namespace bp = boost::python;#endif#include <gflags/gflags.h>#include <glog/logging.h>#include <cstring>#include <map>#include <string>#include <vector>#include "boost/algorithm/string.hpp"#include "caffe/caffe.hpp"#include "caffe/util/signal_handler.h"using caffe::Blob;using caffe::Caffe;using caffe::Net;using caffe::Layer;using caffe::Solver;using caffe::shared_ptr;using caffe::string;using caffe::Timer;using caffe::vector;using std::ostringstream;DEFINE_string(gpu, "", "Optional; run in GPU mode on given device IDs separated by ','." "Use '-gpu all' to run on all available GPUs. The effective training " "batch size is multiplied by the number of devices.");DEFINE_string(solver, "", "The solver definition protocol buffer text file.");DEFINE_string(model, "", "The model definition protocol buffer text file.");DEFINE_string(phase, "", "Optional; network phase (TRAIN or TEST). Only used for 'time'.");DEFINE_int32(level, 0, "Optional; network level.");DEFINE_string(stage, "", "Optional; network stages (not to be confused with phase), " "separated by ','.");DEFINE_string(snapshot, "", "Optional; the snapshot solver state to resume training.");DEFINE_string(weights, "", "Optional; the pretrained weights to initialize finetuning, " "separated by ','. Cannot be set simultaneously with snapshot.");DEFINE_int32(iterations, 50, "The number of iterations to run.");DEFINE_string(sigint_effect, "stop", "Optional; action to take when a SIGINT signal is received: " "snapshot, stop or none.");DEFINE_string(sighup_effect, "snapshot", "Optional; action to take when a SIGHUP signal is received: " "snapshot, stop or none.");// A simple registry for caffe commands.typedef int (*BrewFunction)();typedef std::map<caffe::string, BrewFunction> BrewMap;BrewMap g_brew_map;#define RegisterBrewFunction(func) \namespace { \class __Registerer_##func { \ public: /* NOLINT */ \ __Registerer_##func() { \ g_brew_map[#func] = &func; \ } \}; \__Registerer_##func g_registerer_##func; \}static BrewFunction GetBrewFunction(const caffe::string& name) { if (g_brew_map.count(name)) { return g_brew_map[name]; } else { LOG(ERROR) << "Available caffe actions:"; for (BrewMap::iterator it = g_brew_map.begin(); it != g_brew_map.end(); ++it) { LOG(ERROR) << "\t" << it->first; } LOG(FATAL) << "Unknown action: " << name; return NULL; // not reachable, just to suppress old compiler warnings. }}// Parse GPU ids or use all available devicesstatic void get_gpus(vector<int>* gpus) { if (FLAGS_gpu == "all") { int count = 0;#ifndef CPU_ONLY CUDA_CHECK(cudaGetDeviceCount(&count));#else NO_GPU;#endif for (int i = 0; i < count; ++i) { gpus->push_back(i); } } else if (FLAGS_gpu.size()) { vector<string> strings; boost::split(strings, FLAGS_gpu, boost::is_any_of(",")); for (int i = 0; i < strings.size(); ++i) { gpus->push_back(boost::lexical_cast<int>(strings[i])); } } else { CHECK_EQ(gpus->size(), 0); }}// Parse phase from flagscaffe::Phase get_phase_from_flags(caffe::Phase default_value) { if (FLAGS_phase == "") return default_value; if (FLAGS_phase == "TRAIN") return caffe::TRAIN; if (FLAGS_phase == "TEST") return caffe::TEST; LOG(FATAL) << "phase must be \"TRAIN\" or \"TEST\""; return caffe::TRAIN; // Avoid warning}// Parse stages from flagsvector<string> get_stages_from_flags() { vector<string> stages; boost::split(stages, FLAGS_stage, boost::is_any_of(",")); return stages;}// caffe commands to call by// caffe <command> <args>//// To add a command, define a function "int command()" and register it with// RegisterBrewFunction(action);// Device Query: show diagnostic information for a GPU device.int device_query() { LOG(INFO) << "Querying GPUs " << FLAGS_gpu; vector<int> gpus; get_gpus(&gpus); for (int i = 0; i < gpus.size(); ++i) { caffe::Caffe::SetDevice(gpus[i]); caffe::Caffe::DeviceQuery(); } return 0;}RegisterBrewFunction(device_query);// Load the weights from the specified caffemodel(s) into the train and// test nets.void CopyLayers(caffe::Solver<float>* solver, const std::string& model_list) { std::vector<std::string> model_names; boost::split(model_names, model_list, boost::is_any_of(",") ); for (int i = 0; i < model_names.size(); ++i) { LOG(INFO) << "Finetuning from " << model_names[i]; solver->net()->CopyTrainedLayersFrom(model_names[i]); for (int j = 0; j < solver->test_nets().size(); ++j) { solver->test_nets()[j]->CopyTrainedLayersFrom(model_names[i]); } }}// Translate the signal effect the user specified on the command-line to the// corresponding enumeration.caffe::SolverAction::Enum GetRequestedAction( const std::string& flag_value) { if (flag_value == "stop") { return caffe::SolverAction::STOP; } if (flag_value == "snapshot") { return caffe::SolverAction::SNAPSHOT; } if (flag_value == "none") { return caffe::SolverAction::NONE; } LOG(FATAL) << "Invalid signal effect \""<< flag_value << "\" was specified";}// Train / Finetune a model.int train() { CHECK_GT(FLAGS_solver.size(), 0) << "Need a solver definition to train."; CHECK(!FLAGS_snapshot.size() || !FLAGS_weights.size()) << "Give a snapshot to resume training or weights to finetune " "but not both."; vector<string> stages = get_stages_from_flags(); caffe::SolverParameter solver_param; caffe::ReadSolverParamsFromTextFileOrDie(FLAGS_solver, &solver_param); solver_param.mutable_train_state()->set_level(FLAGS_level); for (int i = 0; i < stages.size(); i++) { solver_param.mutable_train_state()->add_stage(stages[i]); } // If the gpus flag is not provided, allow the mode and device to be set // in the solver prototxt. if (FLAGS_gpu.size() == 0 && solver_param.has_solver_mode() && solver_param.solver_mode() == caffe::SolverParameter_SolverMode_GPU) { if (solver_param.has_device_id()) { FLAGS_gpu = "" + boost::lexical_cast<string>(solver_param.device_id()); } else { // Set default GPU if unspecified FLAGS_gpu = "" + boost::lexical_cast<string>(0); } } vector<int> gpus; get_gpus(&gpus); if (gpus.size() == 0) { LOG(INFO) << "Use CPU."; Caffe::set_mode(Caffe::CPU); } else { ostringstream s; for (int i = 0; i < gpus.size(); ++i) { s << (i ? ", " : "") << gpus[i]; } LOG(INFO) << "Using GPUs " << s.str();#ifndef CPU_ONLY cudaDeviceProp device_prop; for (int i = 0; i < gpus.size(); ++i) { cudaGetDeviceProperties(&device_prop, gpus[i]); LOG(INFO) << "GPU " << gpus[i] << ": " << device_prop.name; }#endif solver_param.set_device_id(gpus[0]); Caffe::SetDevice(gpus[0]); Caffe::set_mode(Caffe::GPU); Caffe::set_solver_count(gpus.size()); } caffe::SignalHandler signal_handler( GetRequestedAction(FLAGS_sigint_effect), GetRequestedAction(FLAGS_sighup_effect)); shared_ptr<caffe::Solver<float> > solver(caffe::SolverRegistry<float>::CreateSolver(solver_param)); solver->SetActionFunction(signal_handler.GetActionFunction()); if (FLAGS_snapshot.size()) { LOG(INFO) << "Resuming from " << FLAGS_snapshot; solver->Restore(FLAGS_snapshot.c_str()); } else if (FLAGS_weights.size()) { CopyLayers(solver.get(), FLAGS_weights); } LOG(INFO) << "Starting Optimization"; if (gpus.size() > 1) {#ifdef USE_NCCL caffe::NCCL<float> nccl(solver); nccl.Run(gpus, FLAGS_snapshot.size() > 0 ? FLAGS_snapshot.c_str() : NULL);#else LOG(FATAL) << "Multi-GPU execution not available - rebuild with USE_NCCL";#endif } else { solver->Solve(); } LOG(INFO) << "Optimization Done."; return 0;}RegisterBrewFunction(train);// Test: score a model.int test() { CHECK_GT(FLAGS_model.size(), 0) << "Need a model definition to score."; CHECK_GT(FLAGS_weights.size(), 0) << "Need model weights to score."; vector<string> stages = get_stages_from_flags(); // Set device id and mode vector<int> gpus; get_gpus(&gpus); if (gpus.size() != 0) { LOG(INFO) << "Use GPU with device ID " << gpus[0];#ifndef CPU_ONLY cudaDeviceProp device_prop; cudaGetDeviceProperties(&device_prop, gpus[0]); LOG(INFO) << "GPU device name: " << device_prop.name;#endif Caffe::SetDevice(gpus[0]); Caffe::set_mode(Caffe::GPU); } else { LOG(INFO) << "Use CPU."; Caffe::set_mode(Caffe::CPU); } // Instantiate the caffe net. Net<float> caffe_net(FLAGS_model, caffe::TEST, FLAGS_level, &stages); caffe_net.CopyTrainedLayersFrom(FLAGS_weights); LOG(INFO) << "Running for " << FLAGS_iterations << " iterations."; vector<int> test_score_output_id; vector<float> test_score; float loss = 0; for (int i = 0; i < FLAGS_iterations; ++i) { float iter_loss; const vector<Blob<float>*>& result = caffe_net.Forward(&iter_loss); loss += iter_loss; int idx = 0; for (int j = 0; j < result.size(); ++j) { const float* result_vec = result[j]->cpu_data(); for (int k = 0; k < result[j]->count(); ++k, ++idx) { const float score = result_vec[k]; if (i == 0) { test_score.push_back(score); test_score_output_id.push_back(j); } else { test_score[idx] += score; } const std::string& output_name = caffe_net.blob_names()[ caffe_net.output_blob_indices()[j]]; LOG(INFO) << "Batch " << i << ", " << output_name << " = " << score; } } } loss /= FLAGS_iterations; LOG(INFO) << "Loss: " << loss; for (int i = 0; i < test_score.size(); ++i) { const std::string& output_name = caffe_net.blob_names()[ caffe_net.output_blob_indices()[test_score_output_id[i]]]; const float loss_weight = caffe_net.blob_loss_weights()[ caffe_net.output_blob_indices()[test_score_output_id[i]]]; std::ostringstream loss_msg_stream; const float mean_score = test_score[i] / FLAGS_iterations; if (loss_weight) { loss_msg_stream << " (* " << loss_weight << " = " << loss_weight * mean_score << " loss)"; } LOG(INFO) << output_name << " = " << mean_score << loss_msg_stream.str(); } return 0;}RegisterBrewFunction(test);// Time: benchmark the execution time of a model.int time() { CHECK_GT(FLAGS_model.size(), 0) << "Need a model definition to time."; caffe::Phase phase = get_phase_from_flags(caffe::TRAIN); vector<string> stages = get_stages_from_flags(); // Set device id and mode vector<int> gpus; get_gpus(&gpus); if (gpus.size() != 0) { LOG(INFO) << "Use GPU with device ID " << gpus[0]; Caffe::SetDevice(gpus[0]); Caffe::set_mode(Caffe::GPU); } else { LOG(INFO) << "Use CPU."; Caffe::set_mode(Caffe::CPU); } // Instantiate the caffe net. Net<float> caffe_net(FLAGS_model, phase, FLAGS_level, &stages); // Do a clean forward and backward pass, so that memory allocation are done // and future iterations will be more stable. LOG(INFO) << "Performing Forward"; // Note that for the speed benchmark, we will assume that the network does // not take any input blobs. float initial_loss; caffe_net.Forward(&initial_loss); LOG(INFO) << "Initial loss: " << initial_loss; LOG(INFO) << "Performing Backward"; caffe_net.Backward(); const vector<shared_ptr<Layer<float> > >& layers = caffe_net.layers(); const vector<vector<Blob<float>*> >& bottom_vecs = caffe_net.bottom_vecs(); const vector<vector<Blob<float>*> >& top_vecs = caffe_net.top_vecs(); const vector<vector<bool> >& bottom_need_backward = caffe_net.bottom_need_backward(); LOG(INFO) << "*** Benchmark begins ***"; LOG(INFO) << "Testing for " << FLAGS_iterations << " iterations."; Timer total_timer; total_timer.Start(); Timer forward_timer; Timer backward_timer; Timer timer; std::vector<double> forward_time_per_layer(layers.size(), 0.0); std::vector<double> backward_time_per_layer(layers.size(), 0.0); double forward_time = 0.0; double backward_time = 0.0; for (int j = 0; j < FLAGS_iterations; ++j) { Timer iter_timer; iter_timer.Start(); forward_timer.Start(); for (int i = 0; i < layers.size(); ++i) { timer.Start(); layers[i]->Forward(bottom_vecs[i], top_vecs[i]); forward_time_per_layer[i] += timer.MicroSeconds(); } forward_time += forward_timer.MicroSeconds(); backward_timer.Start(); for (int i = layers.size() - 1; i >= 0; --i) { timer.Start(); layers[i]->Backward(top_vecs[i], bottom_need_backward[i], bottom_vecs[i]); backward_time_per_layer[i] += timer.MicroSeconds(); } backward_time += backward_timer.MicroSeconds(); LOG(INFO) << "Iteration: " << j + 1 << " forward-backward time: " << iter_timer.MilliSeconds() << " ms."; } LOG(INFO) << "Average time per layer: "; for (int i = 0; i < layers.size(); ++i) { const caffe::string& layername = layers[i]->layer_param().name(); LOG(INFO) << std::setfill(' ') << std::setw(10) << layername << "\tforward: " << forward_time_per_layer[i] / 1000 / FLAGS_iterations << " ms."; LOG(INFO) << std::setfill(' ') << std::setw(10) << layername << "\tbackward: " << backward_time_per_layer[i] / 1000 / FLAGS_iterations << " ms."; } total_timer.Stop(); LOG(INFO) << "Average Forward pass: " << forward_time / 1000 / FLAGS_iterations << " ms."; LOG(INFO) << "Average Backward pass: " << backward_time / 1000 / FLAGS_iterations << " ms."; LOG(INFO) << "Average Forward-Backward: " << total_timer.MilliSeconds() / FLAGS_iterations << " ms."; LOG(INFO) << "Total Time: " << total_timer.MilliSeconds() << " ms."; LOG(INFO) << "*** Benchmark ends ***"; return 0;}RegisterBrewFunction(time);int main(int argc, char** argv) { // Print output to stderr (while still logging). FLAGS_alsologtostderr = 1; // Set version gflags::SetVersionString(AS_STRING(CAFFE_VERSION)); // Usage message. gflags::SetUsageMessage("command line brew\n" "usage: caffe <command> <args>\n\n" "commands:\n" " train train or finetune a model\n" " test score a model\n" " device_query show GPU diagnostic information\n" " time benchmark model execution time"); // Run tool or show usage. caffe::GlobalInit(&argc, &argv); if (argc == 2) {#ifdef WITH_PYTHON_LAYER try {#endif return GetBrewFunction(caffe::string(argv[1]))();#ifdef WITH_PYTHON_LAYER } catch (bp::error_already_set) { PyErr_Print(); return 1; }#endif } else { gflags::ShowUsageWithFlagsRestrict(argv[0], "tools/caffe"); }}
还需要了解一下基本的gflags的基本知识。
- gflags头文件
#include <gflags/gflags.h>
- 初始化所有参数
google::ParseCommandLineFlags(&argc, &argv, flags);
argc和argv就是 main 的入口参数,因为这个函数会改变他们的值,所以都是以指针传入。
第三个参数被称为 flags。如果它是true,ParseCommandLineFlags会从argv中移除标识和它们的参数,相应减少argc的值。然后 argv 只保留命令行参数。
相反, flags是false,ParseCommandLineFlags会保留argc不变,但将会重新调整它们的顺序,使得标识再前面。
- gflags 暂时支持如下参数的类型:
DEFINE_bool: booleanDEFINE_int32: 32-bit integerDEFINE_int64: 64-bit integerDEFINE_uint64: unsigned 64-bit integerDEFINE_double: doubleDEFINE_string: C++ string
提供一个简单的列子
#include <iostream>#include <gflags/gflags.h>#include<map>DEFINE_bool(isvip, false, "If Is VIP");DEFINE_string(ip, "127.0.0.1", "connect ip");DECLARE_int32(port);DEFINE_int32(port, 80, "listen port");int main(int argc, char** argv){ google::ParseCommandLineFlags(&argc, &argv, true); std::cout<<"ip:"<<FLAGS_ip<<std::endl; std::cout<<"port:"<<FLAGS_port<<std::endl;std::cout<<"ip:"<<FLAGS_ip.size()<<std::endl; if (FLAGS_isvip) { std::cout<<"isvip:"<<FLAGS_isvip<<std::endl; } google::ShutDownCommandLineFlags(); return 0;}
ubuntu@ubuntu:~$ g++ d.cpp -o test -lgflags -lpthread ubuntu@ubuntu:~$ ./test ip:127.0.0.1port:80ip:9ubuntu@ubuntu:~$ ./test -ip=123.23.31.41 -port=43ip:123.23.31.41port:43ip:12
基本的语法会使用即可。再看下面这个例子。
#include <iostream>#include <gflags/gflags.h>/** * 定义命令行参数变量 * 默认的主机地址为 127.0.0.1,变量解释为 'the server host' * 默认的端口为 12306,变量解释为 'the server port' */DEFINE_string(host, "127.0.0.1", "the server host");DEFINE_int32(port, 12306, "the server port");int main(int argc, char** argv) { // 解析命令行参数,一般都放在 main 函数中开始位置 gflags::ParseCommandLineFlags(&argc, &argv, true); // 访问参数变量,加上 FLAGS_ std::cout << "The server host is: " << FLAGS_host << ", the server port is: " << FLAGS_port << std::endl; return 0;}
google::ParseCommandLineFlags(&argc, &argv, true);
三个参数的作用:
如果设为true,argv中只保留argv[0],argc会被设置为1。
如果为false,则argv和argc会被保留,注意函数会调整argv中的顺序。
ubuntu@ubuntu:~$ g++ test.cpp -o test -lgflags -lpthread ubuntu@ubuntu:~$ ./test The server host is: 127.0.0.1, the server port is: 12306ubuntu@ubuntu:~$ ./test -host 10.123.78.90 -port 8008The server host is: 10.123.78.90, the server port is: 8008
在学习一下基本的glog 的知识
#include <glog/logging.h>int main(int argc,char* argv[]){ google::InitGoogleLogging(argv[0]); LOG(INFO) << "Hello, GOOGLE!"; // INFO 级别的日志 LOG(ERROR) << "ERROR, GOOGLE!"; // ERROR 级别的日志,直接输出到控制台上(c++ 会介绍stdout 和 stderr 区别) VLOG(10) << "VLOG INFO 10";// 自定义信息 return 0;}
ubuntu@ubuntu:~$ g++ d.cpp -o test -lgflags -lpthread -lglog ubuntu@ubuntu:~$ ./test E0620 20:04:27.647083 10570 d.cpp:8] ERROR, GOOGLE!
如果想把信息都输入到控制台上;
logtostderr (bool, default=false) //是否将所有日志输出到 stderr,而非文件
alsologtostderr(bool,default=false) //是否同时将日志输出到文件和stderr
#include <glog/logging.h>int main(int argc,char* argv[]){ FLAGS_alsologtostderr = 1;//添加这条语句(c++ 会介绍stdout 和 stderr 区别) 会是log(INFO)的信息也输出到控制台上 google::InitGoogleLogging(argv[0]); LOG(INFO) << "Hello, GOOGLE!"; // INFO 级别的日志 LOG(ERROR) << "ERROR, GOOGLE!"; // ERROR 级别的日志,直接输出到控制台上(c++ 会介绍stdout 和 stderr 区别) VLOG(10) << "VLOG INFO 10";// 自定义信息 DLOG(INFO) << "Found cookies";//只有在调试状态下有用 return 0;}
ubuntu@ubuntu:~$ g++ d.cpp -o test -lgflags -lpthread -lglog ubuntu@ubuntu:~$ ./test I0620 20:02:56.162489 10557 d.cpp:7] Hello, GOOGLE!E0620 20:02:56.162962 10557 d.cpp:8] ERROR, GOOGLE!ubuntu@ubuntu:~$
源代码中其他一些函数的含义:
version信息:使用google::SetVersionString设定,使用google::VersionString访问
help信息:使用google::SetUsageMessage设定,使用google::ProgramUsage访问
注意:google::SetUsageMessage和google::SetVersionString必须在google::ParseCommandLineFlags之前执行
caffe::GlobalInit(&argc, &argv);
原函数体在caffe/src/caffe/common.cpp中,
void GlobalInit(int* pargc, char*** pargv) { // Google flags. ::gflags::ParseCommandLineFlags(pargc, pargv, true); // Google logging. ::google::InitGoogleLogging(*(pargv)[0]); // Provide a backtrace on segfault. ::google::InstallFailureSignalHandler();}
这里实现了命令行的解析和初始化 glog
- 五、caffe 之gflags&glogs解析
- 深度学习框架Caffe学习笔记(5)-使用gflags解析命令行参数
- Caffe之WindowDataLayer 解析
- 崩溃之gflags
- caffe之(五)loss层
- gflags
- caffe源码解析之blob
- [Caffe]源码解析之Layer
- Caffe学习之命令行解析
- caffe源码之ImageDataLayer解析
- caffe源码解析之layer
- caffe源码解析之net
- caffe源码解析之solver
- Caffe系列之命令行解析
- CAFFE源码学习笔记之五-internal_thread
- Google gflags库(解析命令行参数)
- 用Google的gflags解析命令行参数
- caffe源码解析之blob(1)
- VS2010 c++编写的程序在别人的机子运行不了,缺少mfc100.dll, xxx100d.dll等的解决方法
- Lua中string.format转义符的使用
- 0-1背包问题动态规划算法
- 单选多选动态加载工具类
- RadditMQ操作
- 五、caffe 之gflags&glogs解析
- 小程序突破五层限制的方法
- android中对观察者模式的使用,Adapter中的观察者
- Linux 升级git 版本
- PHP CURL GET 和 POST 实例
- Linux中安装配置hadoop集群
- cogs729. [网络流24题] 圆桌聚餐
- android反编译静态调试实战2(解密星花和猫友直播)
- rac 搭建流程参考