python --socket（二）粘包

       所谓粘包问题主要还是因为接收方不知道消息之间的界限，不知道一次性提取多少字节的数据所造成的。

此外，发送方引起的粘包是由TCP协议本身造成的，TCP为提

高传输效率，发送方往往要收集到足够多的数据后才发送一个TCP

段。若连续几次需要send的数据都很少，通常TCP会根据优化算

法把这些数据合成一个TCP段后一次发送出去，这样接收方就收到

了粘包数据。

TCP（transport control protocol，传输控制协议）是面向连接

的，面向流的，提供高可靠性服务。收发两端（客户端和服务器端

）都要有一一成对的socket，因此，发送端为了将多个发往接收端

的包，更有效的发到对方，使用了优化方法（Nagle算法），将多

次间隔较小且数据量小的数据，合并成一个大的数据块，然后进行

封包。这样，接收端，就难于分辨出来了，必须提供科学的拆包机

制。 即面向流的通信是无消息保护边界的。
UDP（user datagram protocol，用户数据报协议）是无连接的

，面向消息的，提供高效率服务。不会使用块的合并优化算法， 由

于UDP支持的是一对多的模式，所以接收端的skbuff(套接字缓冲区

）采用了链式结构来记录每一个到达的UDP包，在每个UDP包中

就有了消息头（消息来源地址，端口等信息），这样，对于接收

端来说，就容易进行区分处理了。 即面向消息的通信是有消息保

护边界的。
tcp是基于数据流的，于是收发的消息不能为空，这就需要在客户

端和服务端都添加空消息的处理机制，防止程序卡住，而udp是基

于数据报的，即便是你输入的是空内容（直接回车），那也不是

空消息，udp协议会帮你封装上消息头。

udp的recvfrom是阻塞的，一个recvfrom(x)必须对一个一个

sendinto(y),收完了x个字节的数据就算完成,若是y>x数据就丢失，

这意味着udp根本不会粘包，但是会丢数据，不可靠
tcp的协议数据不会丢，没有收完包，下次接收，会继续上次继续

接收，己端总是在收到ack时才会清除缓冲区内容。数据是可靠的

，但是会粘包。

==============================================

FTP

client

import socket,optparse,json,os,shelveclass FtpClient:    '''Ftp 客户端'''    MSG_SIZE = 1024  # 消息最长1024    def __init__(self):        self.username=None        self.current_dir=None        self.terminal_display=None        self.shelve_obj=shelve.open('.luffy_db')        parser=optparse.OptionParser()        parser.add_option('-s','--server',dest='server',help='ftp server ip_addr')        parser.add_option('-P','--port',type='int',dest='port',help='ftp server port')        parser.add_option("-u", "--username", dest="username", help="username info")        parser.add_option("-p", "--password", dest="password", help="password info")        self.options,self.args=parser.parse_args()        print(self.options,self.args)        self.argv_verification()        self.make_connection()    def argv_verification(self):        '''检查是否提供IP和PORT'''        if  not self.options.server or not self.options.port:            exit('must supply server and port parameters')    def make_connection(self):        '''建立socket链接'''        self.sock=socket.socket(socket.AF_INET,socket.SOCK_STREAM)        self.sock.connect((self.options.server,self.options.port))    def get_response(self):        '''获得服务器返回'''        data=self.sock.recv(self.MSG_SIZE)        return json.loads(data.decode())    def auth(self):        '''登录server'''        count=0        while count<3:            username=input('username: ').strip()            if not username:continue            password=input('password: ').strip()            cmd={                'action_type':'auth',                'username':username,                'password':password            }            self.sock.send(json.dumps(cmd).encode('utf8'))            response=self.get_response()            if response.get('status_code') == 200:                self.username=username                self.terminal_display=self.username                self.current_dir='\\'                return True            else:                print(response.get('status_msg'))            count+=1    def interactive(self):        '''处理与服务器的互动'''        if self.auth():            self.unfinished_file_check()            while True:                user_input =input('[{}]>>: '.format(self.terminal_display)).strip()                if not user_input:continue                cmd_list=user_input.split()                if hasattr(self,'_{}'.format(cmd_list[0])):                    func=getattr(self,'_{}'.format(cmd_list[0]))                    func(cmd_list[1:])    def parameter_checke(self,args,min_args=None,max_args=None,exact_args=None):        '''参数合法性检查'''        if min_args:            if len(args) < min_args:                print('must provide at least {} parameters but {} receivede'.format(min_args,len(args)))                return False        if max_args:            if len(args) > max_args:                print("need at most %s parameters but %s received." % (max_args, len(args)))                return False        if exact_args:            if len(args) != exact_args:                print("need exactly %s parameters but %s received." % (exact_args, len(args)))                return False        return True    def send_msg(self,action_type,**kwargs):        '''打包信息并发送远程'''        msg_data={'action_type':action_type,                  'fill':""}        msg_data.update(kwargs)        bytes_msg=json.dumps(msg_data).encode()        if len(bytes_msg) < self.MSG_SIZE:            msg_data['fill'].zfill(self.MSG_SIZE-len(bytes_msg))            bytes_msg = json.dumps(msg_data).encode()        self.sock.send(bytes_msg)    def get_response(self):        '''收到消息返回解码后结果'''        data=self.sock.recv(self.MSG_SIZE)        return json.loads(data.decode('utf8'))    def unfinished_file_check(self):        '''检查shelve db 展示未下载完的文件        根据用户选择下载或跳过        如要下载，向服务端 发送 文件名 总大小 已下载大小        等待回复        准备下载'''        if list(self.shelve_obj.keys()):            print("-------Unfinished file list -------------")            for index,abs_file in enumerate(self.shelve_obj.keys()):                received_file_size=os.path.getsize(self.shelve_obj[abs_file][1])                print("%s. %s    %s    %s   %s" % (index, abs_file,                                                   self.shelve_obj[abs_file][0],                                                   received_file_size,                                                   received_file_size / self.shelve_obj[abs_file][0] * 100                                                   ))            while True:                choice=input("[select file index to re-download]").strip()                if not choice:continue                if choice == 'back': break                if choice.isdigit():                    choice=int(choice)                    if choice >= 0 and choice <= index:                        selected_file=list(self.shelve_obj.keys())[choice]                        #self.shelve_obj[file_abs_path]=[file_size,'{}.dawnload'.format(filename)]                        received_size = os.path.getsize(self.shelve_obj[selected_file][1])                        self.send_msg('re_get',file_size=self.shelve_obj[selected_file][0],                                      received_size=received_size,                                      abs_filename=selected_file)                        response=self.get_response()                        if response.get('status_code') == 401:                            local_file=self.shelve_obj[selected_file][1]                            f=open(local_file,'ab')                            total_size=self.shelve_obj[selected_file][0]                            current_percent=int(received_size/total_size*100)                            progess_generator=self.progrss_bar(total_size,current_percent=current_percent)                            progess_generator.__next__()                            while received_size > total_size:                                if total_size-received_size<8192:                                    data=self.sock.recv(total_size-received_size)                                else:                                    data=self.sock.recv()                                f.write(data)                                received_size+=len(data)                                progess_generator.send(received_size)                            else:                                print("file re-get done")                                f.close()                        else:                            print(response.get('status_code'))    def _get(self,cmd_args):        '''download file from server        1.拿到文件名        2.发送到服务器        3.等待服务器响应          3.1如果文件存在，拿到文件大小            3.1.1 循环接收          3.2 如果文件不在              print status_code              '''        if self.parameter_checke(cmd_args,min_args=1):            filename=cmd_args[0]            self.send_msg(filename=filename,action_type='get')            response=self.get_response()            if response.get('status_code') == 301:#文件存在                file_size=response.get('file_size')                received_size=0                progress_generator = self.progress_bar(file_size)#生成器                progress_generator.__next__()                file_abs_path=os.path.join(self.current_dir,filename)                self.shelve_obj[file_abs_path]=[file_size,'{}.dawnload'.format(filename)]                f=open('{}.dawnload'.format(filename),'wb')                while received_size < file_size:                    if file_size-received_size<8192:                        data=self.sock.recv(file_size-received_size)                    else:                        data=self.sock.recv(8192)                    f.write(data)                    progress_generator.send(received_size)                else:                    print("---file [%s] recv done,received size [%s]----" % (filename, file_size))                    del self.shelve_obj[file_abs_path]                    f.close()                    os.rename('{}.dawnload'.format(filename),filename)            else:                print(response.get('status_msg'))    def _ls(self,data):        '''查看当前文件夹下的文件        发送命令，接收结果'''        self.send_msg(action_type='ls')        response=self.get_response()        if response.get('status_code') == 302:#准备接收long消息            cmd_result_size=response.get('cmd_result_size')            received_size=0            cmd_result=b''            while received_size < cmd_result_size:                if cmd_result_size -received_size<8192:                    data=self.sock.recv(cmd_result_size -received_size)                else:                    data=self.sock.recv(8192)                received_size+=len(data)                cmd_result+=data            else:               print(cmd_result.decode('GBK'))    def _cd(self,args):        '''检测参数合法性       发送命令和参数       接收返回状态码'''        if self.parameter_checke(args,exact_args=1):            target_dir=args[0]            self.send_msg(action_type='cd',target_dir=target_dir)            response=self.get_response()            if response.get['status_code']== 350:                self.terminal_display=response.get['current_dir']                self.current_dir=response.get['current_dir']    def progrss_bar(self,total_size,current_percent=0,last_percent=0):        '''进度条'''        while True:            received_size=yield            current_percent=int(received_size/total_size*100)            if current_percent >last_percent:                print('#'*(int(current_percent)/2) +'[{}]'.format(current_percent),end='\r')            last_percent=current_percent    def _put(self,data):        '''上传文件到服务器        1.确保文件存在        2.发送文件名和文件大小        3.打开文件开始发送        :param data:        :return:        '''        if self.parameter_checke(data,exact_args=1):            local_file=data[0]            if os.path.isfile(local_file):                total_size=os.path.getsize(local_file)                self.send_msg('put',file_size=total_size,filename=local_file)                f=open(local_file,'rb')                progrss_generator=self.progrss_bar(total_size)                progrss_generator.__next__                uploaded_size=0                for line in f:                    self.sock.send(line)                    uploaded_size+=0                    progrss_generator.send(uploaded_size)                else:                    print()                    print('uploaded file done...'.center(50,'-'))                    f.close()    def _pwd(self,args):        '''        直接打印        :return:        '''        print('{}'.format(self.current_dir))    def _mkdir(self,args):        '''        1.发送命令        2.获得返回        :param args:        :return:        '''        if self.parameter_checke(args, exact_args=1):            new_dir = args[0]            print(new_dir)            self.send_msg(action_type='mkdir', new_dir=new_dir)            response = self.get_response()            if response.get('status_code') == 353:                cmd_result_size = response.get('cmd_result_size')                received_size = 0                cmd_result = b''                while received_size < cmd_result_size:                    if cmd_result_size - received_size < 8192:  # last receive                        data = self.sock.recv(cmd_result_size - received_size)                    else:                        data = self.sock.recv(8192)                    cmd_result += data                    received_size += len(data)                else:                    print(cmd_result.decode("gbk"))            if response.get('status_code') == 352:                print('Dir create success')    def _del(self,args):        '''        发送命令        获得反馈，        '''        if self.parameter_checke(args,min_args=1):            filename=args[0]            self.send_msg(filename=filename,action_type='del')            response=self.get_response()            if response.get('status_code')== 500:                print('delect success')            if response.get('status_code')== 501:                print("NO search file or dir")            else:                data=self.sock.recv(8192)                print(data.decode('GBK'))if __name__ =='__main__':    client=FtpClient()    client.interactive()

server

import sys,osBASE_DIR=os.path.dirname(os.path.dirname(os.path.abspath(__file__)))sys.path.append(BASE_DIR)if __name__ == '__main__':    from core import management    argv_parser=management.ManagementTool(sys.argv)    argv_parser.execute()#执行指令

from core import mainclass ManagementTool:    '''负责对用户输入的指令进行解析并调用相应的模块处理'''    def __init__(self,sys_argv):        self.sys_argv=sys_argv#接收sys.argv的信息['路径','']        print(self.sys_argv)        self.verify_argv()#验证sys_argv指令是否规范    def verify_argv(self):        '''验证指令合法性'''        if len(self.sys_argv) < 2:#如果sys.argv只有一个元素，说明没有输指令            self.help_msg()        cmd=self.sys_argv[1]        if not hasattr(self,cmd):#判断有没有这个功能            print('invalid argument')            self.help_msg()    def help_msg(self):        msg='''        start   start FTP server        stop    stop  FTP server        restart  testart FTP server        createuser username create a ftp user            '''        exit(msg)#退出并打印    def execute(self):        '''解析并执行指令'''        cmd=self.sys_argv[1]        func=getattr(self,cmd)#获取函数对象        func()    def start(self):        '''开启服务器'''        server=main.FTPServer(self)        server.run_forever()

import socket,json,configparser,hashlib,os,subprocess,timefrom conf import settingsclass FTPServer:    '''处理与客户端的所有交互行为'''    STATUS_CODE = {        200: "Passed authentication!",        201: "Wrong username or password!",        300: "File does not exist !",        301: "File exist , and this msg include the file size- !",        302: "This msg include the msg size!",        350: "Dir changed !",        351: "Dir doesn't exist !",        352: "Dir create success",        353:"Dir does exist",        401: "File exist ,ready to re-send !",        402: "File exist ,but file size doesn't match!",        500: 'delect success',        501: "Dir or file doesn't exist !"    }    MSG_SIZE = 1024  # 消息最长1024    def __init__(self,management_instence):        self.management_instence=management_instence        self.sock=socket.socket(socket.AF_INET,socket.SOCK_STREAM)        self.sock.bind((settings.HOST,settings.PORT))        self.sock.listen(5)        self.accounts=self.load_accounts()        self.user_obj=None        self.user_current_dir=None    def run_forever(self):        '''启动服务器,接收连接'''        print('starting FTP server on {},{}'.center(50,'-').format(settings.HOST,settings.PORT))        while True:            self.request,self.addr=self.sock.accept()            print('got a new connection form {}....'.format(self.addr,))            self.handle()    def handle(self):        '''处理与用户的所有指令交互'''        while True:            raw_data=self.request.recv(self.MSG_SIZE)            if not raw_data:                print('connection {} is lost...'.format(self.addr))                del self.request,self.sock                break            data=json.loads(raw_data.decode('utf8'))#解码，反序列化            action_type=data.get('action_type')#从字典中取出action_type的值            if action_type:#判断是否为空                if hasattr(self,'_{}'.format(action_type)):                    func=getattr(self,'_{}'.format(action_type))                    func(data)            else:                print('invalid command')    def load_accounts(self):        '''加载所有帐户信息'''        config_obj=configparser.ConfigParser()        config_obj.read(settings.ACCOUNTS_FILE)#读取文件        return config_obj    def authenticate(self,username,password):        '''用户认证方法'''        if username in self.accounts:            _password = self.accounts[username]['password']            md5_obj=hashlib.md5()            md5_obj.update(password.encode())            if _password == md5_obj.hexdigest():                self.user_obj=self.accounts[username]#拿到客户在服务器上的所有信息                self.user_obj['home']=os.path.join(settings.USER_HOME_DIR,username)                self.user_current_dir=self.user_obj['home']                #增加家目录                return True            else:                return False        else:            return False    def send_response(self,status_code,*args,**kwargs):        '''打包发送信息给客户端        :param status_code:200        :param args:        :param kwargs:{'filename':filename,'size':filesize}        :return        '''        data=kwargs#变成字典，如果有值        data['status_code']=status_code        data['status_msg']=self.STATUS_CODE[status_code]        data['fill']=''        bytes_data=bytes(json.dumps(data).encode())        if len(bytes_data) < self.MSG_SIZE:            data['fill'].zfill(self.MSG_SIZE-len(bytes_data))            bytes_data = bytes(json.dumps(data).encode())        self.request.send(bytes_data)    def _auth(self,data):        '''处理用户登录认证'''        if self.authenticate(data.get('username'),data.get('password')):            print('pass auth...')            self.send_response(status_code=200)        else:            self.send_response(status_code=201)    def _get(self,data):        '''        1.收到文件名        2.判断文件是否存在            2.1 文件存在，返回文件大小和状态码                2.1.1 发送文件            2.2 文件不存在，返回状态码        :return:        '''        filename=data.get('filename')        full_path=os.path.join(self.user_obj['home'],filename)#控制用户只能在家目录下寻找文件        if os.path.isfile(full_path):            filesize=os.stat(full_path).st_size            self.send_response(status_code=301,file_size=filesize)            print('ready to send file')            f=open(filename,'rb')            for line in f:                self.request.send(line)            else:                print('file send done..',full_path)            f.close()        else:            self.send_response(status_code=300)#文件不存在    def _ls(self,data):        '''实现dir命令并返回结果给client'''        print(self.user_current_dir)        cmd_obj=subprocess.Popen('dir {}'.format(self.user_current_dir),shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)        stdout=cmd_obj.stdout.read()        stderr=cmd_obj.stderr.read()        cmd_result=stdout+stderr        if not cmd_result:            cmd_result=b'current dir has not file at all'        self.send_response(status_code=302,cmd_result_size=len(cmd_result))        self.request.sendall(cmd_result)    def _cd(self,data):        '''实现让用户觉得在切换文件夹的假象        1.根据 用户发送过来target_dir 与 user_current_dir 拼接        2.检测有没有target_dir            2.1 如果有就改变user_current_dir            2.2 如果没有就返回错误值        '''        target_dir=data.get('target_dir')        full_dir=os.path.abspath(os.path.join(self.user_current_dir,target_dir))        #abspath 是为了解决 cd ..  c:\a\b\c\..\..=c:\a        if os.path.isdir(full_dir):            if full_dir.startswith(self.user_obj['home']):                self.user_current_dir=full_dir                relative_current_dir=full_dir.replace(self.user_obj['home'],'')                self.send_response(status_code=350,current_dir=relative_current_dir)            else:                self.send_response(status_code=351)        else:            self.send_response(status_code=351)    def _put(self,data):        '''        1.接收文件名和文件大小        2.检测有没有同名文件            2.1如果有，给上传文件改名                2.1 准备接收            2.2 没有，直接接收        :param data:        :return:        '''        local_file=data.get('filename')        full_path=os.path.join(self.user_current_dir,local_file)        if os.path.isfile(full_path):            filename='{}.{}'.format(full_path,time.time())        else:            filename=full_path        f=open(filename,'wb')        received_size=0        total_size=data.get('tatal_size')        while received_size < total_size:            if total_size-received_size <8192:                data=self.request.recv(total_size-received_size)            else:                data=self.request.recv(8192)            received_size+=len(data)            f.write(data)        else:            print('file %s recv done'% local_file)            f.close()    def _re_get(self,args):        '''        1.拿到文件名，拼接路径        2.判断文件在不在            2.1 在，判断大小是否一样                2.1.1 大小一样，返回状态码                    2.1.1.1 根据对方已经收到的大小，sock到位置继续传                2.1.2 不一样，返回状态码            2.2 不在 返回状态码        :param args:        :return:        '''        abs_filename=args.get('abs_filename')        full_path=os.path.join(self.user_obj['home'],abs_filename.strip('\\'))        if os.path.isfile(full_path):            if os.path.getsize(full_path)== args.get('file_size'):                self.send_response(status_code=401)                f=open(full_path,'rb')                f.seek(args.get('received_size'))                for line in f:                    self.request.recv(line)                else:                    print("-----file re-send done------")                    f.close()            else:                self.send_response(status_code=402,file_size_on_server=os.path.getsize(full_path))        else:            self.send_response(status_code=300)    def _mkdir(self,argv):        '''        1.拼接路径        2.实现mkdir 命令        3.返回结果        :param argv:        :return:        '''        new_dir=argv.get('new_dir')        create_target_dir=os.path.join(self.user_current_dir,new_dir)        cmd_obj = subprocess.Popen('mkdir {}'.format(create_target_dir), shell=True, stdout=subprocess.PIPE,                                   stderr=subprocess.PIPE)        stdout = cmd_obj.stdout.read()        stderr = cmd_obj.stderr.read()        cmd_result = stdout + stderr        if len(cmd_result)==0:            self.send_response(status_code=352)        else:            self.send_response(status_code=353,cmd_result_size=len(cmd_result))            self.request.sendall(cmd_result)    def _del(self,args):        '''        1.拼接路径        2.判断是文件还是文件夹            2.1 文件 使用del            2.2 DIR  使用 rd        3.返回消息        :param args:        :return:        '''        filename=args.get('filename')        file_path=os.path.join(self.user_current_dir,filename)        if os.path.isfile(file_path):            s = subprocess.Popen('del {}'.format(file_path), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)            f1 = s.stdout.read()            f2 = s.stderr.read()            f = f1 + f2            if not f:                self.send_response(status_code=500)            else:                self.request.sendall(f)        elif os.path.isdir(file_path):            s = subprocess.Popen('rd {}'.format(file_path), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)            f1 = s.stdout.read()            f2 = s.stderr.read()            f = f1 + f2            if not f:                self.send_response(status_code=500)            else:                self.request.sendall(f)        else:            self.send_response(status_code=501)