nginx 拦截请求信息与响应信息并切割日志

来源：互联网发布：淘宝店铺权重是什么编辑：程序博客网时间：2024/06/09 19:31

1、请求唯一标示request_id跟踪(nginx支持32位唯一标示)
2、设置日志文件及级别(我是跟踪业务请求日志,error_log,error级别)
3、通过body_filter_by_lua_file 指令来拦截设置需要的请求响应信息(转换成通用json格式)
4、通过log_by_lua写入业务日志文件

直接上文件
1、nginx.conf

#运行用户和组,缺省为nobody,若改为别的用户和组，则需要先创建用户和组
#user wls81 wls;

#开启进程数,一般与CPU核数等同
worker_processes 4;

#设置进程到cpu(四cpu：0001 0010 0100 1000)
#worker_cpu_affinity 0001 0010 0100 1000;

#每个进程最大打开文件数
worker_rlimit_nofile 8000;

#进程号保存文件
#pid /wls/apache/applogs/ng_sbtps-opf-dmzweb-nginx/nginx.pid;
error_log /wls/appsystems/nginx/nginx/logs/error.log;
pid /wls/appsystems/nginx/nginx/logs/nginx.pid;

#设置错误日志
#error_log /wls/apache/applogs/ng_sbtps-opf-dmzweb-nginx/error.log;
error_log logs/error.log notice;
error_log logs/error.log info;
error_log logs/error.log debug;

events
{
#运行模式设置[ kqueue | rtsig | epoll | /dev/poll | select | poll ];
#使用epoll(linux2.6的高性能方式)
use epoll;

#每个进程最大连接数(最大连接=连接数x进程数)
worker_connections 8000;
}

http
{

#文件扩展名与文件类型映射表
#include mime.types;

#默认文件类型
#default_type text/html;
default_type application/octet-stream;

#服务器名称相关设置
server_names_hash_max_size 256;
server_names_hash_bucket_size 512;

#默认编码
charset UTF-8;

#开启高效文件传输模式,直接从系统级别传输(Linux 2.4以上支持,纯文件服务器才能打开)
sendfile off;

#网络TCP_NOPUSH和TCP_NODELAY参数设置
#tcp_nopush on;
tcp_nodelay on;

#设置保留链接超时时间为75秒设置header超时时间为20秒
keepalive_timeout 75 20;

#打开gzip压缩
gzip on;

#最小压缩文件大小
gzip_min_length 1K;

#压缩缓冲区
gzip_buffers 4 8k;

#压缩类型
gzip_types text/* text/css application/javascript application/x-javascript application/xml;

#压缩级别 1-9 1最快 9最慢
gzip_comp_level 9;

#压缩通过代理的所有文件
gzip_proxied any;

#vary header支持
gzip_vary on;

#压缩版本（默认1.1，前端为squid2.5使用1.0）
gzip_http_version 1.1;


#输出缓冲区
output_buffers 4 32k;

#输出拆包大小
postpone_output 1460;

#接收header的缓冲区大小
client_header_buffer_size 128k;
large_client_header_buffers 4 256k;

#客户端发送header超时
client_header_timeout 3m;

#客户端发送内容超时
client_body_timeout 3m;

#发送到客户端超时
send_timeout 3m;

#捕捉代理端的http错误
#proxy_intercept_errors on;

jvm_path '/wls/appsystems/jdk1.8.0_25/jre/lib/amd64/server/libjvm.so';

jvm_classpath '/wls/appsystems/jdk1.8.0_25/lib/*:/wls/appsystems/nginx/libs/nginx-clojure/openapi-nginx-java.jar';

#init初始化handle
#jvm_handler_type 'java';
#jvm_init_handler_name 'com.pa.openapi.nginx.JVMInitHandler';

#日志文件格式
log_format main '$remote_addr $http_x_forwarded_for $remote_user $time_iso8601 $status '
'$server_protocol logTraceId:$request_id $comp_sign $request_method $reSetReqUri $uri $http_referer $gzip_ratio '
'"$http_user_agent" '
'$body_bytes_sent $bytes_sent $request_length "$upstream_addr" "$upstream_header_time" "$upstream_response_time" $request_time';
log_format requestBody '$remote_addr - $remote_user [$time_local] "$request" '
'"$status" $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for" logTraceId:$request_id req_body:"$request_body" resp_body:"$resp_body" resp_map:"$resp_map"'
'$upstream_addr $bytes_sent $request_length "$upstream_response_time" "$request_time"';
lua_need_request_body on;
#日志文件(不记录)
#access_log /dev/null;
#access_log logs/access.log main;

include /wls/appsystems/nginx/nginx/conf/servers/respCodeMap.map;
#api映射配置
lua_shared_dict apiMappingShared 50m;
lua_shared_dict healthStatus 1m;
client_body_buffer_size 8m;
client_max_body_size 8m;
init_by_lua_file /wls/appsystems/nginx/nginx/conf/scripts/luas/init.lua;
#lua_package_path "/wls/appsystems/nginx/nginxmodule/lua-resty-redis-cluster-master/lib/resty/rediscluster.lua;;";
#lua_package_cpath "/wls/appsystems/nginx/luajit/lib/lua/5.1/?.so;;";
#获取访问客户的真实IP
map $http_x_forwarded_for $clientRealIp {
"" $remote_addr;
~^(?P<firstAddr>[0-9\.]+),?.*$$firstAddr;
}

#默认主机配置
#include default_host.conf;

upstream SBTPS-OPF-AIO-WEB-SF-AUTH-APP-ITS
{
#tomcat的地址和端口
server 10.25.174.28:32050;
}

server
{
#设置监听端口
listen 37775 default;

#设置服务器域名(IP访问和多域名访问可不设置)
#server_name _*;
#server_name 192.168.23.23;
#server_name www.test.com;

#error_log /wls/appsystems/nginx/nginx/logs/error.log debug_http;
access_log /wls/appsystems/nginx/nginx/logs/access.log main;
#access_log /wls/appsystems/nginx/nginx/logs/requestBody.log requestBody;
error_log /wls/appsystems/nginx/nginx/logs/request.log error;

##定义$request_trace_id的值，在1.11.x之前，我们可以使用类似的方式声明，只要能确保
##其值出现重复的可能性尽可能的小即可。
set $request_trace_id trace-id-$pid-$connection-$bytes_sent-$msec;
set $comp_sign "";
set $reSetReqUri $request_uri;
set $resp_body "";
set $resp_map "";

#设定根目录(若全部请求转发到后端服务器则不需要设置)

error_page 404 = /httpStatusRewrite?apiRespCodeParam=-10201;
error_page 403 = /httpStatusRewrite?apiRespCodeParam=-5;
error_page 503 = /httpStatusRewrite?apiRespCodeParam=-6;
error_page 500 502 504 = /httpStatusRewrite?apiRespCodeParam=-1;

location /httpStatusRewrite{
default_type "text/plain";
set $apiRespCode $arg_apiRespCodeParam;
if ( $apiRespMsg = '' ) {
return 200 '{"resCode":"-1","resMsg":"请求失败"}';
}
return 200 '{"resCode":"$apiRespCode","resMsg":"$apiRespMsg"}';
}

#location /sssssss
#{
# 加上他会读取POST请求数据，不需要的时候建议关掉
#always_read_body on;
#web根目录，根据实际情况调整
#proxy_set_header Host $host;
#proxy_set_header X-real-ip $remote_addr;
#proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

#rewrite_handler_type 'java';
#rewrite_handler_name 'com.pa.openapi.nginx.RewriteHandler';
#proxy_pass http://$comp_sign/AuthApp$reSetReqUri;
#}
#location /springboot
#{
#default_type "text/plain";
#set $apiSign $arg_apiSign;
#proxy_pass http://localhost:8888/nginx/redis/forward_sbtpsboa_IPOsOrderCancel;
#proxy_pass http://localhost:8888/nginx/redis/$apiSign;
#}

#设定根目录(若全部请求转发到后端服务器则不需要设置)
location /
{
#web根目录，根据实际情况调整
proxy_set_header Host $host;
proxy_set_header X-real-ip $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header logTraceId $request_id;
set $comp_sign AUTH2;
set $reSetReqUri $request_uri;
lua_need_request_body on;
#set $resp_map "";
rewrite_by_lua 'apiMapping.getCompSign()';
body_filter_by_lua_file /wls/appsystems/nginx/nginx/conf/scripts/luas/modules/apimapping/request.lua;
proxy_set_header X-Request-ID $request_id;
proxy_pass http://$comp_sign/AuthApp$reSetReqUri;
#log_by_lua_file /wls/appsystems/nginx/nginx/conf/scripts/luas/modules/apimapping/log.lua;
log_by_lua 'ngx.log(ngx.ERR,"resp_map is :"..ngx.var.resp_map)';

}

#location /test
#{
#content_by_lua '
#local config = {
#name = "test",
#serv_list = {
#{ip="10.25.174.131", port = 4000},
#{ip="10.25.174.131", port = 4100},
#{ip="10.25.174.131", port = 4200},
#{ip="10.25.174.131", port = 4300},
#{ip="10.25.174.131", port = 4400},
#{ip="10.25.174.131", port = 4500},
#{ip="10.25.174.131", port = 4100},
#{ip="10.25.174.131", port = 4200},
#}
#}
#local redis_cluster = require "resty.rediscluster"
#local red = redis_cluster:new(config)

#for i = 1, 2 do
#red:init_pipeline()

#red:set("dog", "an animal")
#red:get("dog")
#red:set("dog", "hello")
#red:get("dog")

#local results = red:commit_pipeline()
#local cjson = require "cjson"
#ngx.log(ngx.ERR,"result is :"..cjson.encode(results))
#ngx.say(cjson.encode(results))
#end

#red:close()
#';
#}

}

#包含其它虚拟主机配置;
#include servers/*.com;
include servers/*.net;
include servers/*.org;
include servers/*.cn;
}

2、body_filter_by_lua_file request.lua

local resp_body = string.sub(ngx.arg[1], 1, 1000)
ngx.ctx.buffered = (ngx.ctx.buffered or"") .. resp_body
if ngx.arg[2] then
ngx.var.resp_body = ngx.ctx.buffered
end
local map = {}
map["logTraceId"] = ngx.var.request_id;
map["remote_addr"] = ngx.var.remote_addr;
map["remote_user"] = ngx.var.remote_user;
if pcall(decodeURI,ngx.var.time_local)then
map["time_local"] = decodeURI(ngx.var.time_local);
end
if pcall(decodeURI,ngx.var.request) then
map["request"] = decodeURI(ngx.var.request);
end
map["status"] = ngx.var.status;
map["body_bytes_sent"] = ngx.var.body_bytes_sent;
map["http_referer"] = ngx.var.http_referer;
map["http_user_agent"] = ngx.var.http_user_agent;
map["http_x_forwarded_for"] = ngx.var.http_x_forwarded_for;
if pcall(decodeURI,ngx.var.request_body) then
map["request_body"] = decodeURI(ngx.var.request_body);
end
if pcall(decodeURI,ngx.var.resp_body) then
map["resp_body"] = decodeURI(ngx.var.resp_body);
end
map["upstream_addr"] = ngx.var.upstream_addr;
map["bytes_sent"] = ngx.var.bytes_sent;
map["request_length"] = ngx.var.request_length;
map["upstream_response_time"] = ngx.var.upstream_response_time;
map["request_time"] = ngx.var.request_time;
ngx.var.resp_map = cjson.encode(map);

3、切割脚本

#!/bin/bash

#setting log path
log_files_path="/wls/appsystems/nginx/nginx/logs/"
log_files_backup_path="/wls/appsystems/nginx/nginx/backup_logs"
log_files_dir=${log_files_path}/$(date +"%Y%m")/$(date +"%d")

#setting nginx
nginx_sbin="/wls/appsystems/nginx/nginx/sbin/nginx"

if [ ! -d $log_files_dir ]; then
mkdir -p $log_files_dir
fi

cd $log_files_path

#setting log name
log_files_name_request=(request)
log_files_name_access=(access)
log_files_num_request=${#log_files_name_request[@]}
log_files_num_access=${#log_files_name_access[@]}

for((i=0;i<$log_files_num_request;i++));do
mv ${log_files_path}${log_files_name_request[i]}.log ${log_files_dir}/${log_files_name_request[i]}.log
/bin/sed -e 's/x22/"/g' ${log_files_dir}/${log_files_name_request[i]}.log > ${log_files_dir}/${log_files_name_request[i]}.$(date +%Y-%m-%d-%H).log
/bin/rm ${log_files_dir}/${log_files_name_request[i]}.log

mv ${log_files_path}${log_files_name_access[i]}.log ${log_files_dir}/${log_files_name_access[i]}.log
/bin/sed -e 's/x22/"/g' ${log_files_dir}/${log_files_name_access[i]}.log > ${log_files_dir}/${log_files_name_access[i]}.$(date +%Y-%m-%d-%H).log
/bin/rm ${log_files_dir}/${log_files_name_access[i]}.log
done

kill -USR1 `cat /wls/appsystems/nginx/nginx/logs/nginx.pid`

阅读全文

0 0