Question

似乎如果我的程序尝试写入完整的文件系统，它最初会出错，并且“设备上没有剩余空间”。几乎立即，但如果我让它运行一分钟左右，它会减慢几个数量级。

请注意，这是一个最小的测试用例，这种行为首先在Java日志框架中被注意到，因此小的（512字节）输出块和很多它们。

main.c

#include <stdlib.h>
#include <stdio.h>
#include <stdbool.h>
#include <stddef.h>
#include <time.h>
#include <sys/time.h>


long microTime() {
    struct timeval time;
    gettimeofday(&time, NULL);

    return time.tv_sec * 1000 * 1000 + time.tv_usec;
}

int main(int argc, const char * argv[]) {

    char *payload ;
    payload = (char *)malloc(512 * sizeof(char));

    if (payload == NULL) {
        printf("Failed to alloc memory\n");
        exit(1);
    }

    FILE *fp;
    fp = fopen(argv[1], "a+");

    printf("opened [%s]\n", argv[1]);

    if (fp == NULL) {
        printf("Failed to open [%s]\n", argv[1]);
        exit(1);
    }

    for (;;) {
        int batchSize = 100000;
        bool errored = false;
        long runStart = microTime();
        for (int i = 0; i < batchSize; i ++) {
            size_t result = fwrite(payload, 512 * sizeof(char), 1, fp);
            if (result == 0 && !errored) {
                perror("Failed to write to disk");
                errored = true;
            }

        }
        long runEnd = microTime();
        printf("total elapsed %dms\n", (int)(runEnd - runStart) / 1000);
    }

    return 0;
}

（请原谅我的C，这可能是我近20年来写的第一个C程序）

运行：

gcc -std=c99 main.c && ./a.out /path/to/somewhere/file1.bin

警告此程序将填满您的磁盘分区

输出：

total elapsed 42ms
total elapsed 105ms
total elapsed 104ms
total elapsed 125ms
... skip until the disk fills
Failed to write to disk: No space left on device
total elapsed 104ms
Failed to write to disk: No space left on device
total elapsed 76ms
Failed to write to disk: No space left on device
total elapsed 84ms
Failed to write to disk: No space left on device
... then skip a little more about one minute
total elapsed 8096ms
Failed to write to disk: No space left on device
total elapsed 43245ms
Failed to write to disk: No space left on device
total elapsed 48670ms
Failed to write to disk: No space left on device
total elapsed 45929ms
Failed to write to disk: No space left on device

我的期望是这个程序会以相当低的恒定时间运行，以便写入磁盘。

我在当地的centos 6.4 vagrant box，amazon linux和ubuntu 14.04 ec2 box上运行它，结果完全相同。有趣的是，在OSX 10.9.5尝试填充磁盘映像时似乎没有发生这种情况。

所以我的问题是，是什么导致这种明显的限制？

更新：使用strace -t -T

运行

10:27:38 write(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 4096) = 4096 <0.000011>
10:27:38 write(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 4096) = 4096 <0.000011>
10:27:38 write(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 4096) = -1 ENOSPC (No space left on device) <0.000066>
10:27:38 dup(2)                         = 4 <0.000006>
10:27:38 fcntl(4, F_GETFL)              = 0x8002 (flags O_RDWR|O_LARGEFILE) <0.000011>
10:27:38 fstat(4, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 0), ...}) = 0 <0.000006>
10:27:38 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa8e97f2000 <0.000009>
10:27:38 lseek(4, 0, SEEK_CUR)          = -1 ESPIPE (Illegal seek) <0.000005>
10:27:38 write(4, "Failed to write to disk: No spac"..., 49Failed to write to disk: No space left on device
) = 49 <0.000006>
10:27:38 close(4)                       = 0 <0.000006>
10:27:38 munmap(0x7fa8e97f2000, 4096)   = 0 <0.000015>
10:27:38 write(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 4096) = -1 ENOSPC (No space left on device) <0.000026>
10:27:38 write(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 4096) = -1 ENOSPC (No space left on device) <0.000017>
10:27:38 write(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 4096) = -1 ENOSPC (No space left on device) <0.000016>
10:27:38 write(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 4096) = -1 ENOSPC (No space left on device) <0.000016>
10:27:38 write(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 4096) = -1 ENOSPC (No space left on device) <0.000015>
10:27:38 write(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 4096) = -1 ENOSPC (No space left on device) <0.000016>
10:27:38 write(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 4096) = -1 ENOSPC (No space left on device) <0.000015>
10:27:38 write(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 4096) = -1 ENOSPC (No space left on device) <0.000015>
10:27:38 write(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 4096) = -1 ENOSPC (No space left on device) <0.000016>
10:27:38 write(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 4096) = -1 ENOSPC (No space left on device) <0.000016>

... skipping to the end
10:30:02 write(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 4096) = -1 ENOSPC (No space left on device) <0.005747>
10:30:02 write(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 4096) = -1 ENOSPC (No space left on device) <0.005231>
10:30:02 write(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 4096) = -1 ENOSPC (No space left on device) <0.005496>
10:30:02 write(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 4096) = -1 ENOSPC (No space left on device) <0.005870>
10:30:02 write(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 4096) = -1 ENOSPC (No space left on device) <0.005823>
10:30:02 write(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 4096) = -1 ENOSPC (No space left on device) <0.005841>

write()的通话时间不是来自~0.00001s =＆gt;在运行结束时约为0.005秒。

The full run.log is 250mb

更新2：为各个阶段添加CPU使用情况详细信息：

盯着跑步，即填满磁盘

top - 11:00:56 up  2:52,  2 users,  load average: 1.79, 0.99, 0.48
Tasks:  75 total,   3 running,  72 sleeping,   0 stopped,   0 zombie
Cpu(s):  7.3%us, 71.8%sy,  0.0%ni,  0.0%id,  0.0%wa, 14.6%hi,  6.3%si,  0.0%st
Mem:    603764k total,   561868k used,    41896k free,   134976k buffers
Swap:  1254392k total,        0k used,  1254392k free,   359020k cached

  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
 9861 vagrant   20   0  4056  500  412 R 54.5  0.1   0:01.88 a.out
  766 root      20   0     0    0    0 R 36.9  0.0   0:28.51 flush-8:0
   28 root      20   0     0    0    0 S  5.6  0.0   0:05.14 kswapd0
   16 root      20   0     0    0    0 S  2.3  0.0   4:51.07 kblockd/0

磁盘已满，快速错误

top - 11:01:11 up  2:52,  2 users,  load average: 1.68, 1.01, 0.49
Tasks:  75 total,   2 running,  73 sleeping,   0 stopped,   0 zombie
Cpu(s):  9.0%us, 91.0%sy,  0.0%ni,  0.0%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
Mem:    603764k total,   555424k used,    48340k free,   134976k buffers
Swap:  1254392k total,        0k used,  1254392k free,   352224k cached

  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
 9861 vagrant   20   0  4056  552  464 R 99.5  0.1   0:12.91 a.out
  988 root      20   0  215m 1572  860 S  0.3  0.3   0:05.05 VBoxService
    1 root      20   0 19228 1348 1072 S  0.0  0.2   0:00.24 init
    2 root      20   0     0    0    0 S  0.0  0.0   0:00.00 kthreadd

慢速错误

top - 11:03:03 up  2:54,  2 users,  load average: 1.63, 1.14, 0.59
Tasks:  74 total,   3 running,  71 sleeping,   0 stopped,   0 zombie
Cpu(s):  0.0%us,  0.4%sy,  0.0%ni,  0.0%id, 98.8%wa,  0.4%hi,  0.4%si,  0.0%st
Mem:    603764k total,   555284k used,    48480k free,   134976k buffers
Swap:  1254392k total,        0k used,  1254392k free,   352308k cached

  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
  216 root      20   0     0    0    0 R  3.7  0.0   4:50.72 jbd2/sda1-8
   16 root      20   0     0    0    0 R  3.3  0.0   4:53.04 kblockd/0
 9861 vagrant   20   0  4056  552  464 D  1.3  0.1   1:17.47 a.out
    1 root      20   0 19228 1348 1072 S  0.0  0.2   0:00.24 init

Answer 1

当write系统调用阻塞并变为同步时，您可能正在观察Linux虚拟内存的影响。这是因为您的应用程序不断生成要写入磁盘的数据，并且磁盘无法快速存储数据。

或者，后台刷新在磁盘填满后的某个时间启动，以便write系统调用与内核刷新线程争用文件系统内部结构的访问权限，这样返回时间会更长{ {1}}。

请参阅Better Linux Disk Caching & Performance with vm.dirty_ratio & vm.dirty_background_ratio：

vm.dirty_background_ratio是可以用“脏”页面填充的系统内存的百分比 - 仍然需要写入磁盘的内存页面 - 在pdflush / flush / kdmflush后台进程启动以将其写入磁盘之前。我的例子是10％，所以如果我的虚拟服务器有32 GB的内存，那就是3.2 GB的数据，可以在完成任务之前就坐在RAM中。

vm.dirty_ratio是在必须将所有内容提交到磁盘之前可以用脏页填充的绝对最大系统内存量。 当系统到达此点时，所有新的I / O块都会被阻塞，直到脏页被写入磁盘。这通常是长I / O暂停的来源，但是可以防止太多数据在内存中被不安全地缓存。

什么限制fwrite（）调用linux上的完整磁盘？

1 个答案: