linux读写二进制大文件

https://raw.githubusercontent.com/ceph/ceph/master/src/common/safe_io.c

一般而言，用pwrite和pread读写磁盘文件不需要用循环，但是在读写超大的文件时就一定需要循环，保证正确读写

// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-// vim: ts=8 sw=2 smarttab/* * Ceph - scalable distributed file system * * Copyright (C) 2011 New Dream Network * * This is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License version 2.1, as published by the Free Software * Foundation.  See file COPYING. * */#define _XOPEN_SOURCE 500#include <stdio.h>#include <string.h>#include <unistd.h>#include <errno.h>#include <fcntl.h>#include <limits.h>#include "common/safe_io.h"#include "include/compat.h"ssize_t safe_read(int fd, void *buf, size_t count){size_t cnt = 0;while (cnt < count) {ssize_t r = read(fd, buf, count - cnt);if (r <= 0) {if (r == 0) {// EOFreturn cnt;}if (errno == EINTR)continue;return -errno;}cnt += r;buf = (char *)buf + r;}return cnt;}ssize_t safe_read_exact(int fd, void *buf, size_t count){        ssize_t ret = safe_read(fd, buf, count);if (ret < 0)return ret;if ((size_t)ret != count)return -EDOM;return 0;} ssize_t safe_write(int fd, const void *buf, size_t count){while (count > 0) {ssize_t r = write(fd, buf, count);if (r < 0) {if (errno == EINTR)continue;return -errno;}count -= r;buf = (char *)buf + r;}return 0;}ssize_t safe_pread(int fd, void *buf, size_t count, off_t offset){size_t cnt = 0;char *b = (char*)buf;while (cnt < count) {ssize_t r = pread(fd, b + cnt, count - cnt, offset + cnt);if (r <= 0) {if (r == 0) {// EOFreturn cnt;}if (errno == EINTR)continue;return -errno;}cnt += r;}return cnt;}ssize_t safe_pread_exact(int fd, void *buf, size_t count, off_t offset){ssize_t ret = safe_pread(fd, buf, count, offset);if (ret < 0)return ret;if ((size_t)ret != count)return -EDOM;return 0;}ssize_t safe_pwrite(int fd, const void *buf, size_t count, off_t offset){while (count > 0) {ssize_t r = pwrite(fd, buf, count, offset);if (r < 0) {if (errno == EINTR)continue;return -errno;}count -= r;buf = (char *)buf + r;offset += r;}return 0;}#ifdef CEPH_HAVE_SPLICEssize_t safe_splice(int fd_in, loff_t *off_in, int fd_out, loff_t *off_out,    size_t len, unsigned int flags){  size_t cnt = 0;  while (cnt < len) {    ssize_t r = splice(fd_in, off_in, fd_out, off_out, len - cnt, flags);    if (r <= 0) {      if (r == 0) {// EOFreturn cnt;      }      if (errno == EINTR)continue;      return -errno;    }    cnt += r;  }  return cnt;}ssize_t safe_splice_exact(int fd_in, loff_t *off_in, int fd_out,  loff_t *off_out, size_t len, unsigned int flags){  ssize_t ret = safe_splice(fd_in, off_in, fd_out, off_out, len, flags);  if (ret < 0)    return ret;  if ((size_t)ret != len)    return -EDOM;  return 0;}#endifint safe_write_file(const char *base, const char *file,    const char *val, size_t vallen){  int ret;  char fn[PATH_MAX];  char tmp[PATH_MAX];  int fd;  // does the file already have correct content?  char oldval[80];  ret = safe_read_file(base, file, oldval, sizeof(oldval));  if (ret == (int)vallen && memcmp(oldval, val, vallen) == 0)    return 0;  // yes.  snprintf(fn, sizeof(fn), "%s/%s", base, file);  snprintf(tmp, sizeof(tmp), "%s/%s.tmp", base, file);  fd = open(tmp, O_WRONLY|O_CREAT|O_TRUNC, 0644);  if (fd < 0) {    ret = errno;    return -ret;  }  ret = safe_write(fd, val, vallen);  if (ret) {    VOID_TEMP_FAILURE_RETRY(close(fd));    return ret;  }  ret = fsync(fd);  if (ret < 0) ret = -errno;  VOID_TEMP_FAILURE_RETRY(close(fd));  if (ret < 0) {    unlink(tmp);    return ret;  }  ret = rename(tmp, fn);  if (ret < 0) {    ret = -errno;    unlink(tmp);    return ret;  }  fd = open(base, O_RDONLY);  if (fd < 0) {    ret = -errno;    return ret;  }  ret = fsync(fd);  if (ret < 0) ret = -errno;  VOID_TEMP_FAILURE_RETRY(close(fd));  return ret;}int safe_read_file(const char *base, const char *file,   char *val, size_t vallen){  char fn[PATH_MAX];  int fd, len;  snprintf(fn, sizeof(fn), "%s/%s", base, file);  fd = open(fn, O_RDONLY);  if (fd < 0) {    return -errno;  }  len = safe_read(fd, val, vallen);  if (len < 0) {    VOID_TEMP_FAILURE_RETRY(close(fd));    return len;  }  // close sometimes returns errors, but only after write()  VOID_TEMP_FAILURE_RETRY(close(fd));  return len;}

下面是自己写的代码，为了验证read/writeBlock的正确性，分两批写入文件，然后又分两批读入文件，实际工程中只要一次读写就行了

#include <unistd.h>#include <iostream>#include <stdlib.h>#include <string.h>#include <stdint.h>#include <string>#include <sys/types.h>#include <sys/stat.h>#include <fcntl.h>#include <fstream>#include <vector>#include <sstream>#include <errno.h>using namespace std;bool writeBlock(int fd, int64_t &fdOft,int8_t *arr, int64_t len) {    int64_t num = 0;    int64_t oft = 0;    int8_t *arr1 = (int8_t*)arr;    while (oft < len) {        num = pwrite(fd, arr1+oft, len-oft, fdOft+oft);        if (num == -1) {            return false;        }        oft += num;    }    fdOft += oft;    return true;}bool readBlock(int fd, int64_t &fdOft, void *arr, int64_t len) {    int64_t num = 0;    int64_t oft = 0;    int8_t *b = (int8_t*)arr;    while (oft < len) {        num = pread(fd, b + oft, len - oft, fdOft + oft);        if (num <= 0) {            if (num == 0) {                if (oft != len) return false;                else return true;            }            if (errno == EINTR) {                continue;//the call was interrupted by a signal befor any data was read            }            return false;        }        oft += num;    }    fdOft += oft;    return true;}int main(int argc, char *argv[]) {    int64_t len = atol(argv[1]);    cout << "len = "<<len <<endl;    int fd = open(argv[2], O_RDWR|O_CREAT|O_TRUNC, S_IRWXU);    if (fd == -1) {        cout << "not open\n";        return -1;    }    int64_t total = sizeof(double)*len;    cout << "total = " << total <<endl;    double *arr = (double*)malloc(total);    if (arr == NULL) {        cout << "malloc err\n";        return -1;    }    arr[0] = 0.001;    for (int64_t i = 1; i < len; ++i) {        arr[i] = arr[i-1] + 0.001;    }    //int64_t num = pwrite(fd,arr,total,0);    int64_t fdOft = 0;    int64_t first = total/2;    int64_t second = total - first;    bool flag = writeBlock(fd, fdOft,(int8_t*)arr, first);    if (!flag) {        cout << "first no\n";    }    else {        cout << "first yes\n";    }flag = writeBlock(fd, fdOft, (int8_t*)arr + first, second);    //cout << num<<endl;if (!flag) {        cout << "second no\n";    }    else {        cout << "second yes\n";    }close(fd);    int fdr = open(argv[2], O_RDONLY);    double *arr1 = (double*)malloc(total);    if (arr1 == NULL) {        cout << "malloc error\n";        return -1;    }        int64_t readOft = 0;    if (readBlock(fdr, readOft, arr1, first)) {        cout << "first read yes\n";    }    if (readBlock(fdr, readOft, (int8_t*)arr1 + first, second)) {        cout << "second read yes\n";    }    flag = true;    for (int64_t i = 0; i < len; ++i) {        if (arr1[i] != arr[i]) {            flag = false;            cout << i << " no equal\n";        }          }    if (flag) cout << "equal\n";    close(fdr);    return 1;}