lua 异步HTTPS并发请求库

项目使用skynet框架，这个框架主要用lua写逻辑，但缺乏对HTTPS支持，所以我利用一点时间写了lua模块，支持异步HTTPS请求，文章这里讲述HTTPS相关知识，如何接入openssl请求HTTPS数据，同时也分享了lua模块给大家参考。

HTTPS说明

HTTPS可以理解成 HTTP协议的安全版，协议还是HTTP协议，只是对传输过程的数据进行了加密处理，保证数据传输的安全。（默认端口是443）

HTTPS验证数据的过程如下：

skynet HTTPS支持

skynet只封装了HTTP的接口，没有对HTTPS做支持，所以要外接lua 库使用。

skynet支持HTTPS请求有两种方法：

 优点缺点CURL支持http/https/ftp等，接入较简单并发支持差OpenSSL更底层，效率较高接入复杂

其实，CURL是利用OpenSSL实现的，有网友封装了非阻塞版本的lua CURL库，可用于skynet处理HTTPS请求。链接猛击这里。

前面提到了CURL的缺点，CURL本身可做并发请求（libcurl multi），但做法却是将所有URL请求合到一起处理，需等全部URL数据处理完毕才返回数据。假设其中一个URL出现超时，那一起的其他URL都会受影响。

所以，文章推荐使用OpenSSL，这里先介绍C/C++如何处理HTTPS请求，然后再封装一个lua库，给大家演示下 skynet 如何请求HTTPS数据。

C/C++处理HTTPS请求

这里以一个例子，说明C/C++如何处理HTTPS请求。

#include <stdio.h>#include <string.h>#include <errno.h>#include <sys/socket.h>#include <resolv.h>#include <stdlib.h>#include <netinet/in.h>#include <arpa/inet.h>#include <unistd.h>#include <sys/types.h>#include <sys/stat.h>#include <fcntl.h>#include "openssl/ssl.h"#include "openssl/err.h"#define MAXBUF 1024int main(int argc, char **argv){int sockfd, len;char sendFN[1024];struct sockaddr_in dest;char buffer[MAXBUF + 1];SSL_CTX *ctx;SSL *ssl;char host_file[] = "";char  host_addr[] = "www.jd.com";char ip[] = "112.91.125.129";int port = "443";/* SSL 库初始化 */SSL_library_init();OpenSSL_add_all_algorithms();SSL_load_error_strings();ctx = SSL_CTX_new(SSLv23_client_method());if (ctx == NULL){ERR_print_errors_fp(stdout);exit(1);}if ((sockfd = socket(AF_INET, SOCK_STREAM, 0)) < 0){perror("Socket Create Fail!");exit(errno);}/* 建立 TCP 连接 */bzero(&dest, sizeof(dest));dest.sin_family = AF_INET;dest.sin_port = htons(atoi(port));if (inet_aton(ip, (struct in_addr *) &dest.sin_addr.s_addr) == 0){perror("Socket Init Fail!");exit(errno);}printf("Socket Created\n");if (connect(sockfd, (struct sockaddr *) &dest, sizeof(dest)) != 0){perror("Socket Connect Fail!");exit(errno);}printf("Socket Connected\n");/* 绑定 Socket 与 SSL */ssl = SSL_new(ctx);SSL_set_fd(ssl, sockfd);/* 建立 SSL 连接 */if (SSL_connect(ssl) == -1)ERR_print_errors_fp(stderr);elseprintf("SSL Connected with %s encryption\n", SSL_get_cipher(ssl));sprintf(sendFN, "GET /%s HTTP/1.1\r\nHost: %s\r\nConnection: Close\r\n\r\n", host_file, host_addr);/* SSL 发数据 */len = SSL_write(ssl, sendFN, strlen(sendFN));if (len < 0)printf("SSL Send failure! errno = %d, err_msg = %s\n", errno, strerror(errno));printf("SSL Send Done !\n\n");bzero(buffer, MAXBUF + 1);int nbytes;/* SSL 收数据 */while ((nbytes = SSL_read(ssl, buffer, 1)) == 1) {/* 打印收到的数据 */printf("%s", buffer);}printf("\n\nSSL Read Done !\n");/* 关闭连接 */SSL_shutdown(ssl);SSL_free(ssl);close(sockfd);SSL_CTX_free(ctx);return 0;}

原理很简单，底层还是走了socket，只是TCP连接建立后，把连接交给SSL，让SSL收发数据。

Lua处理HTTPS请求

这里提供lua版本处理HTTPS请求，除了skynet，其他lua项目也可以使用。

以下是lua C模块的代码，保存为lua_httpsc.c

#include <stdlib.h>#include <string.h>#include <lua.h>#include <lauxlib.h>#include <time.h>#include <netinet/in.h>#include <sys/types.h>#include <sys/socket.h>#include <arpa/inet.h>#include <unistd.h>#include <errno.h>#include <fcntl.h>#include "openssl/ssl.h"#include "openssl/err.h"#include <poll.h>#define CACHE_SIZE 0x1000#define ERROR_FD -1#define SEND_RETRY 10static SSL_CTX *ctx = NULL;typedef struct {int is_init;} cutil_conf_t;enum conn_st{CONNECT_INIT = 1,CONNECT_PORT = 2,CONNECT_SSL = 3,CONNECT_DONE = 4};typedef struct {int fd;SSL* ssl;enum conn_st status;} cutil_fd_t;static cutil_conf_t* fetch_config(lua_State *L) {cutil_conf_t* cfg;cfg = lua_touserdata(L, lua_upvalueindex(1));if (!cfg)luaL_error(L, "httpsc: Unable to fetch cfg");return cfg;}static void close_fd_t(cutil_fd_t* fd_t) {if ( fd_t == NULL )return;SSL* ssl = fd_t->ssl;if ( ssl != NULL ) {SSL_shutdown(ssl);SSL_free(ssl);}int fd = fd_t->fd;if (fd != ERROR_FD)close(fd);free(fd_t);}static int try_connect_ssl(SSL* ssl) {int ret,err;ret = SSL_connect(ssl);err = SSL_get_error(ssl, ret);if (ret == 1) return 0;if (err != SSL_ERROR_WANT_WRITE && err != SSL_ERROR_WANT_READ ) {return -1;}return 1;}static int lconnect(lua_State *L) {cutil_conf_t* cfg = fetch_config(L);if(!cfg->is_init){luaL_error(L, "httpsc: Not inited");return 0;}const char * addr = luaL_checkstring(L, 1);int port = luaL_checkinteger(L, 2);cutil_fd_t* fd_t = (cutil_fd_t *)malloc(sizeof(cutil_fd_t));if ( fd_t == NULL )return luaL_error(L, "httpsc: Create fd %s %d failed", addr, port);fd_t->fd = ERROR_FD;fd_t->ssl = NULL;fd_t->status = CONNECT_INIT;int fd = socket(AF_INET, SOCK_STREAM, 0);struct sockaddr_in my_addr;fd_t->fd = fd;bzero(&my_addr, sizeof(my_addr));my_addr.sin_addr.s_addr = inet_addr(addr);my_addr.sin_family = AF_INET;my_addr.sin_port = htons(port);int ret;struct timeval timeo = {3, 0};socklen_t len = sizeof(timeo);ret = setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &timeo, len);if (ret) {close_fd_t(fd_t);return luaL_error(L, "httpsc: Setsockopt %s %d failed", addr, port);}int flag = fcntl(fd, F_GETFL, 0);fcntl(fd, F_SETFL, flag | O_NONBLOCK);ret = connect(fd, (struct sockaddr *)&my_addr, sizeof(struct sockaddr_in));if (ret != 0) {if (errno == EINPROGRESS) {fd_t->status = CONNECT_PORT;} else {close_fd_t(fd_t);return luaL_error(L, "httpsc: Connect %s %d failed", addr, port);}} else {fd_t->status = CONNECT_SSL;SSL *ssl = SSL_new(ctx);fd_t->ssl = ssl;SSL_set_fd(ssl, fd);ret = try_connect_ssl(ssl);if (ret == 0) {fd_t->status = CONNECT_DONE;} else if (ret == -1) {close_fd_t(fd_t);return luaL_error(L, "httpsc ssl_connect error, errno = %d", errno);}}lua_pushlightuserdata(L, fd_t);return 1;}static int lcheck_connect(lua_State *L) {cutil_fd_t* fd_t = (cutil_fd_t* ) lua_touserdata(L, 1);if ( fd_t == NULL )return luaL_error(L, "httpsc fd error");switch (fd_t->status) {case CONNECT_DONE:lua_pushboolean(L, 1);return 1;case CONNECT_PORT: {struct pollfd fds;int ret, err;fds.fd = fd_t->fd;fds.events = POLLIN | POLLOUT;/* get status immediately */ret = poll(&fds, 1, 0);if (ret != -1) {socklen_t len = sizeof(int);ret = getsockopt(fd_t->fd, SOL_SOCKET, SO_ERROR, &err, &len);if (ret < 0) {close_fd_t(fd_t);return luaL_error(L, "httpsc getsockopt error, ret = %d", ret);}if (err == 0) {fd_t->status = CONNECT_SSL;SSL *ssl = SSL_new(ctx);fd_t->ssl = ssl;SSL_set_fd(ssl, fd_t->fd);ret = try_connect_ssl(ssl);if (ret == 0) {fd_t->status = CONNECT_DONE;lua_pushboolean(L, 1);return 1;} else if (ret == -1) {close_fd_t(fd_t);return luaL_error(L, "httpsc connect ssl error, errno = %d", errno);}} else {if (errno == EAGAIN || errno == EINTR || errno == EINPROGRESS ) {return 0;} else {close_fd_t(fd_t);return luaL_error(L, "httpsc connect sockopt error, errno = %d", errno);}}} else {close_fd_t(fd_t);return luaL_error(L, "httpsc connect poll error, ret = %d", ret);}return 0;}case CONNECT_SSL:{int ret = try_connect_ssl(fd_t->ssl);if (ret == 0) {fd_t->status = CONNECT_DONE;lua_pushboolean(L, 1);return 1;} else if (ret == -1) {close_fd_t(fd_t);return luaL_error(L, "httpsc connect ssl 2 error, errno = %d", errno);}return 0;}default:;}close_fd_t(fd_t);return luaL_error(L, "httpsc connect fator error");}static int lclose(lua_State *L) {cutil_fd_t* fd_t = (cutil_fd_t* ) lua_touserdata(L, 1);if ( fd_t == NULL )return luaL_error(L, "httpsc fd error");close_fd_t(fd_t);return 0;}static void block_send(lua_State *L, SSL *ssl, const char * buffer, int sz) {int retry = SEND_RETRY;while(sz > 0) {if (retry <= 0) {luaL_error(L, "httpsc: socket error: retry timeout");return;}retry -= 1;int r = SSL_write(ssl, buffer, sz);if (r < 0) {if (errno == EAGAIN || errno == EINTR)continue;luaL_error(L, "httpsc: socket error: %s", strerror(errno));return;}buffer += r;sz -= r;}}static int lsend(lua_State *L) {cutil_conf_t* cfg = fetch_config(L);if(!cfg->is_init){luaL_error(L, "httpsc: Not inited");return 0;}size_t sz = 0;cutil_fd_t* fd_t = (cutil_fd_t* ) lua_touserdata(L, 1);if ( fd_t == NULL )return luaL_error(L, "httpsc fd error");SSL* ssl = fd_t->ssl;const char * msg = luaL_checklstring(L, 2, &sz);block_send(L, ssl, msg, (int)sz);return 0;}static int lrecv(lua_State *L) {cutil_conf_t* cfg = fetch_config(L);if(!cfg->is_init){luaL_error(L, "httpsc: Not inited");return 0;}cutil_fd_t* fd_t = (cutil_fd_t* ) lua_touserdata(L, 1);if ( fd_t == NULL )return luaL_error(L, "httpsc fd error");SSL* ssl = fd_t->ssl;int top = lua_gettop(L);char buffer[CACHE_SIZE];int size = CACHE_SIZE;if ( top > 1 && lua_isnumber(L, 2)) {int _size = lua_tointeger(L, 2);size = _size > size ? size : _size;}int r = SSL_read(ssl, buffer, size);// if (r == 0) {// lua_pushliteral(L, "");// /* close */// return 1;// }if (r <= 0) {if (errno == EAGAIN || errno == EINTR) {return 0;}return luaL_error(L, "httpsc: socket error: %s", strerror(errno));}lua_pushlstring(L, buffer, r);return 1;}static int lusleep(lua_State *L) {int n = luaL_checknumber(L, 1);usleep(n);return 0;}/* GC, clean up the buf */static int _gc(lua_State *L){cutil_conf_t *cfg;cfg = lua_touserdata(L, 1);if (ctx != NULL){SSL_CTX_free(ctx);ctx = NULL;}/* todo: auto gc */cfg = NULL;return 0;}static void _create_config(lua_State *L){cutil_conf_t *cfg;cfg = lua_newuserdata(L, sizeof(*cfg));cfg->is_init = !!NULL;/* Create GC method to clean up buf */lua_newtable(L);lua_pushcfunction(L, _gc);lua_setfield(L, -2, "__gc");lua_setmetatable(L, -2);/* openssl init */if ( ctx == NULL) {SSL_library_init();OpenSSL_add_all_algorithms();SSL_load_error_strings();ctx = SSL_CTX_new(SSLv23_client_method());if (ctx == NULL){ERR_print_errors_fp(stdout);luaL_error(L, "httpsc: Unable to init openssl");return;}}cfg->is_init = !NULL;}int luaopen_httpsc(lua_State *L){static const luaL_Reg funcs[] = {{ "connect", lconnect },{ "check_connect", lcheck_connect },{ "recv", lrecv },{ "send", lsend },{ "close", lclose },{ "usleep", lusleep },{NULL, NULL}};lua_newtable(L);_create_config(L);luaL_setfuncs(L, funcs, 1);return 1;}

编译到项目执行后，lua调用的方法如下：

local httpsc = require "httpsc"for k,v in pairs(httpsc ) doprint(k,v)endlocal fd = httpsc.connect("163.177.151.109", 443)print(fd)while true dolocal ok = httpsc.check_connect(fd)if ok then break endhttpsc.usleep(10000)end-- httpsc.usleep(10000)httpsc.send(fd, "GET / HTTP/1.1\r\nAccept: */*\r\nHost: www.baidu.com\r\nConnection: Close\r\n\r\n")httpsc.usleep(1000000)local body = ""while true dolocal r = httpsc.recv(fd)print(r)if r and #r>0 thenbody = body .. relsebreakendendprint(body)httpsc.close(fd)print("ok!")

以上只是一个简单的例子，我同时也参考skynet clientsocket写了一个对HTTP协议支持友好的版本，代码托管在Git : https://github.com/chenweiqi/lua_httpsc ，欢迎交流。

这个lua代码目前有个不足，没有做openssl多线程支持，不支持同时多个线程加载和调用模块，这个以后的版本会做优化

OpenSSL动态库的编译

既然基于OpenSSL开发，就需要编译OpenSSL库来使用。

这里提供最近版本几个依赖库的编译方法：

openssl  - libssl.a / libcrypto.a

wget https://www.openssl.org/source/openssl-1.0.2k.tar.gztar -zxf openssl-1.0.2k.tar.gzcd  openssl-1.0.2k./config -fPIC enable-sharedmake dependmakemake installfind -name "*.a"

zlib - zlib.a

wget http://zlib.net/zlib-1.2.11.tar.xztar -zxf zlib-1.2.11.tar.xzcd zlib-1.2.11export CFLAGS=" -fPIC"./configuremakefind -name "*.a"

idn - libidn.a

wget http://ftp.gnu.org/gnu/libidn/libidn-1.33.tar.gztar -zxf libidn-1.33.tar.gzcd libidn-1.33export CFLAGS=" -fPIC"./configuremakefind -name "*.a"

更新说明：

2017/4/15 优化socket创建连接过程，由阻塞改成非阻塞方式

2017/6/9 标题由“skynet lua 支持HTTPS请求”改成“lua 异步HTTPS并发请求库”

参考：

http://blog.csdn.net/mycwq/article/details/64440253