Предположим, что я пишу пару файлов на диск, между 2 МБ и 5 ГБ. Каковы разумные значения буфера для FileStream?
Можно ли работать с буферизацией в несколько мегабайт, или я должен придерживаться килобайтных буферов?
С# FileStream: оптимальный размер буфера для записи больших файлов?
Ответ 1
Размер буфера по умолчанию - 4 KiB.
Кроме того, посмотрите здесь: Шаблоны последовательного программирования и производительность с .NET
Ответ 2
Быстрый небольшой тест, основанный на упомянутом документе, не показывает увеличения производительности в моей системе, превышающей размер буфера 128 Кбайт. Ваш пробег может отличаться, не стесняйтесь использовать ниже.
Stopwatch sw = new Stopwatch();
Random rand = new Random(); // seed a random number generator
int numberOfBytes = 2 << 22; //8,192KB File
byte nextByte;
for (int i = 1; i <= 28; i++) //Limited loop to 28 to prevent out of memory
{
sw.Start();
using (FileStream fs = new FileStream(
String.Format(@"C:\TEMP\TEST{0}.DAT", i), // name of file
FileMode.Create, // create or overwrite existing file
FileAccess.Write, // write-only access
FileShare.None, // no sharing
2 << i, // block transfer of i=18 -> size = 256 KB
FileOptions.None))
{
for (int j = 0; j < numberOfBytes; j++)
{
nextByte = (byte)(rand.Next() % 256); // generate a random byte
fs.WriteByte(nextByte); // write it
}
}
sw.Stop();
Console.WriteLine(String.Format("Buffer is 2 << {0} Elapsed: {1}", i, sw.Elapsed));
sw.Reset();
}
Ответ 3
TL;DR: Размер буфера в большинстве случаев не имеет значения. Микро-оптимизировать что-то еще. Просто не используйте буфер ниже 128 КБ, если вы установили useAsync = true в FileStream.
Был немного раздражен тем, что не было однозначного ответа на этот вопрос. Итак, вот журнал из теста, который я создал. Тест проводился на вторичном SSD, чтобы избежать искажения результатов ОС и общего использования.
"Файл" и "Буфер" в байтах.
useAsync = false:
Test Name: FileWritePerformance
Test Outcome: Passed
Result StandardOutput:
File: 1024, Buffer: 1, Elapsed: 00:00:00.0021467
File: 1024, Buffer: 2, Elapsed: 00:00:00.0011164
File: 1024, Buffer: 4, Elapsed: 00:00:00.0012392
File: 1024, Buffer: 8, Elapsed: 00:00:00.0010978
File: 1024, Buffer: 16, Elapsed: 00:00:00.0014263
File: 1024, Buffer: 32, Elapsed: 00:00:00.0010724
File: 1024, Buffer: 64, Elapsed: 00:00:00.0012182
File: 1024, Buffer: 128, Elapsed: 00:00:00.0013605
File: 1024, Buffer: 256, Elapsed: 00:00:00.0010234
File: 1024, Buffer: 512, Elapsed: 00:00:00.0011247
File: 1024, Buffer: 1024, Elapsed: 00:00:00.0013137
File: 1024, Buffer: 2048, Elapsed: 00:00:00.0013527
File: 1024, Buffer: 4096, Elapsed: 00:00:00.0010895
File: 1024, Buffer: 8192, Elapsed: 00:00:00.0017128
File: 1024, Buffer: 16384, Elapsed: 00:00:00.0011272
File: 1024, Buffer: 32768, Elapsed: 00:00:00.0010547
File: 1024, Buffer: 65536, Elapsed: 00:00:00.0011427
File: 1024, Buffer: 131072, Elapsed: 00:00:00.0011571
File: 1024, Buffer: 262144, Elapsed: 00:00:00.0010978
File: 1024, Buffer: 524288, Elapsed: 00:00:00.0010884
File: 1024, Buffer: 1048576, Elapsed: 00:00:00.0011504
File: 1024, Buffer: 2097152, Elapsed: 00:00:00.0011308
File: 1024, Buffer: 4194304, Elapsed: 00:00:00.0026100
File: 1024, Buffer: 8388608, Elapsed: 00:00:00.0015160
File: 1024, Buffer: 16777216, Elapsed: 00:00:00.0011654
File: 1024, Buffer: 33554432, Elapsed: 00:00:00.0013214
File: 1024, Buffer: 67108864, Elapsed: 00:00:00.0012531
File: 1024, Buffer: 134217728, Elapsed: 00:00:00.0031671
File: 1024, Buffer: 268435456, Elapsed: 00:00:00.0032789
File: 1024, Buffer: 536870912, Elapsed: 00:00:00.0049508
File: 1024, Buffer: 1073741824, Elapsed: 00:00:00.0079707
File: 1048576, Buffer: 1, Elapsed: 00:00:00.0175447
File: 1048576, Buffer: 2, Elapsed: 00:00:00.0213337
File: 1048576, Buffer: 4, Elapsed: 00:00:00.0139657
File: 1048576, Buffer: 8, Elapsed: 00:00:00.0135683
File: 1048576, Buffer: 16, Elapsed: 00:00:00.0124353
File: 1048576, Buffer: 32, Elapsed: 00:00:00.0226043
File: 1048576, Buffer: 64, Elapsed: 00:00:00.0123077
File: 1048576, Buffer: 128, Elapsed: 00:00:00.0130151
File: 1048576, Buffer: 256, Elapsed: 00:00:00.0130583
File: 1048576, Buffer: 512, Elapsed: 00:00:00.0130555
File: 1048576, Buffer: 1024, Elapsed: 00:00:00.0130760
File: 1048576, Buffer: 2048, Elapsed: 00:00:00.0111780
File: 1048576, Buffer: 4096, Elapsed: 00:00:00.0097345
File: 1048576, Buffer: 8192, Elapsed: 00:00:00.0096204
File: 1048576, Buffer: 16384, Elapsed: 00:00:00.0085635
File: 1048576, Buffer: 32768, Elapsed: 00:00:00.0081099
File: 1048576, Buffer: 65536, Elapsed: 00:00:00.0076785
File: 1048576, Buffer: 131072, Elapsed: 00:00:00.0069143
File: 1048576, Buffer: 262144, Elapsed: 00:00:00.0078431
File: 1048576, Buffer: 524288, Elapsed: 00:00:00.0065928
File: 1048576, Buffer: 1048576, Elapsed: 00:00:00.0096246
File: 1048576, Buffer: 2097152, Elapsed: 00:00:00.0094253
File: 1048576, Buffer: 4194304, Elapsed: 00:00:00.0089587
File: 1048576, Buffer: 8388608, Elapsed: 00:00:00.0073527
File: 1048576, Buffer: 16777216, Elapsed: 00:00:00.0075993
File: 1048576, Buffer: 33554432, Elapsed: 00:00:00.0069752
File: 1048576, Buffer: 67108864, Elapsed: 00:00:00.0087534
File: 1048576, Buffer: 134217728, Elapsed: 00:00:00.0089197
File: 1048576, Buffer: 268435456, Elapsed: 00:00:00.0072583
File: 1048576, Buffer: 536870912, Elapsed: 00:00:00.0077418
File: 1048576, Buffer: 1073741824, Elapsed: 00:00:00.0217475
File: 1073741824, Buffer: 1, Elapsed: 00:00:08.6674554
File: 1073741824, Buffer: 2, Elapsed: 00:00:08.3731788
File: 1073741824, Buffer: 4, Elapsed: 00:00:08.8015734
File: 1073741824, Buffer: 8, Elapsed: 00:00:08.4734106
File: 1073741824, Buffer: 16, Elapsed: 00:00:08.8242336
File: 1073741824, Buffer: 32, Elapsed: 00:00:08.3961399
File: 1073741824, Buffer: 64, Elapsed: 00:00:08.6481375
File: 1073741824, Buffer: 128, Elapsed: 00:00:08.4404953
File: 1073741824, Buffer: 256, Elapsed: 00:00:08.6811610
File: 1073741824, Buffer: 512, Elapsed: 00:00:08.3297239
File: 1073741824, Buffer: 1024, Elapsed: 00:00:08.9151381
File: 1073741824, Buffer: 2048, Elapsed: 00:00:06.7781371
File: 1073741824, Buffer: 4096, Elapsed: 00:00:05.7471923
File: 1073741824, Buffer: 8192, Elapsed: 00:00:04.6344858
File: 1073741824, Buffer: 16384, Elapsed: 00:00:04.4647610
File: 1073741824, Buffer: 32768, Elapsed: 00:00:04.2576851
File: 1073741824, Buffer: 65536, Elapsed: 00:00:04.3291236
File: 1073741824, Buffer: 131072, Elapsed: 00:00:04.1955776
File: 1073741824, Buffer: 262144, Elapsed: 00:00:04.5730469
File: 1073741824, Buffer: 524288, Elapsed: 00:00:05.1157704
File: 1073741824, Buffer: 1048576, Elapsed: 00:00:05.5262273
File: 1073741824, Buffer: 2097152, Elapsed: 00:00:05.6662534
File: 1073741824, Buffer: 4194304, Elapsed: 00:00:05.6285232
File: 1073741824, Buffer: 8388608, Elapsed: 00:00:05.5857300
File: 1073741824, Buffer: 16777216, Elapsed: 00:00:05.6386955
File: 1073741824, Buffer: 33554432, Elapsed: 00:00:05.6467390
File: 1073741824, Buffer: 67108864, Elapsed: 00:00:05.6969160
File: 1073741824, Buffer: 134217728, Elapsed: 00:00:05.7044284
File: 1073741824, Buffer: 268435456, Elapsed: 00:00:05.8279939
File: 1073741824, Buffer: 536870912, Elapsed: 00:00:05.9805622
File: 1073741824, Buffer: 1073741824, Elapsed: 00:00:06.1534688
useAsync = true:
Test Name: FileWritePerformance
Test Outcome: Passed
Result StandardOutput:
File: 1024, Buffer: 1, Elapsed: 00:00:00.0034314
File: 1024, Buffer: 2, Elapsed: 00:00:00.0026302
File: 1024, Buffer: 4, Elapsed: 00:00:00.0024232
File: 1024, Buffer: 8, Elapsed: 00:00:00.0009797
File: 1024, Buffer: 16, Elapsed: 00:00:00.0047272
File: 1024, Buffer: 32, Elapsed: 00:00:00.0010624
File: 1024, Buffer: 64, Elapsed: 00:00:00.0010657
File: 1024, Buffer: 128, Elapsed: 00:00:00.0039249
File: 1024, Buffer: 256, Elapsed: 00:00:00.0011540
File: 1024, Buffer: 512, Elapsed: 00:00:00.0011479
File: 1024, Buffer: 1024, Elapsed: 00:00:00.0010245
File: 1024, Buffer: 2048, Elapsed: 00:00:00.0011083
File: 1024, Buffer: 4096, Elapsed: 00:00:00.0011263
File: 1024, Buffer: 8192, Elapsed: 00:00:00.0009716
File: 1024, Buffer: 16384, Elapsed: 00:00:00.0011319
File: 1024, Buffer: 32768, Elapsed: 00:00:00.0035012
File: 1024, Buffer: 65536, Elapsed: 00:00:00.0010170
File: 1024, Buffer: 131072, Elapsed: 00:00:00.0010995
File: 1024, Buffer: 262144, Elapsed: 00:00:00.0010381
File: 1024, Buffer: 524288, Elapsed: 00:00:00.0010018
File: 1024, Buffer: 1048576, Elapsed: 00:00:00.0011629
File: 1024, Buffer: 2097152, Elapsed: 00:00:00.0011969
File: 1024, Buffer: 4194304, Elapsed: 00:00:00.0021307
File: 1024, Buffer: 8388608, Elapsed: 00:00:00.0015213
File: 1024, Buffer: 16777216, Elapsed: 00:00:00.0012500
File: 1024, Buffer: 33554432, Elapsed: 00:00:00.0012102
File: 1024, Buffer: 67108864, Elapsed: 00:00:00.0013962
File: 1024, Buffer: 134217728, Elapsed: 00:00:00.0036005
File: 1024, Buffer: 268435456, Elapsed: 00:00:00.0034270
File: 1024, Buffer: 536870912, Elapsed: 00:00:00.0070516
File: 1024, Buffer: 1073741824, Elapsed: 00:00:00.0094566
File: 1048576, Buffer: 1, Elapsed: 00:00:00.1161675
File: 1048576, Buffer: 2, Elapsed: 00:00:00.0867371
File: 1048576, Buffer: 4, Elapsed: 00:00:00.0938809
File: 1048576, Buffer: 8, Elapsed: 00:00:00.0891819
File: 1048576, Buffer: 16, Elapsed: 00:00:00.0856688
File: 1048576, Buffer: 32, Elapsed: 00:00:00.0829121
File: 1048576, Buffer: 64, Elapsed: 00:00:00.1021716
File: 1048576, Buffer: 128, Elapsed: 00:00:00.0819636
File: 1048576, Buffer: 256, Elapsed: 00:00:00.0870802
File: 1048576, Buffer: 512, Elapsed: 00:00:00.0938601
File: 1048576, Buffer: 1024, Elapsed: 00:00:00.0906152
File: 1048576, Buffer: 2048, Elapsed: 00:00:00.0203416
File: 1048576, Buffer: 4096, Elapsed: 00:00:00.0156412
File: 1048576, Buffer: 8192, Elapsed: 00:00:00.0188322
File: 1048576, Buffer: 16384, Elapsed: 00:00:00.0090570
File: 1048576, Buffer: 32768, Elapsed: 00:00:00.0089590
File: 1048576, Buffer: 65536, Elapsed: 00:00:00.0111345
File: 1048576, Buffer: 131072, Elapsed: 00:00:00.0074629
File: 1048576, Buffer: 262144, Elapsed: 00:00:00.0087473
File: 1048576, Buffer: 524288, Elapsed: 00:00:00.0090963
File: 1048576, Buffer: 1048576, Elapsed: 00:00:00.0106754
File: 1048576, Buffer: 2097152, Elapsed: 00:00:00.0072628
File: 1048576, Buffer: 4194304, Elapsed: 00:00:00.0072143
File: 1048576, Buffer: 8388608, Elapsed: 00:00:00.0108595
File: 1048576, Buffer: 16777216, Elapsed: 00:00:00.0078417
File: 1048576, Buffer: 33554432, Elapsed: 00:00:00.0078852
File: 1048576, Buffer: 67108864, Elapsed: 00:00:00.0101042
File: 1048576, Buffer: 134217728, Elapsed: 00:00:00.0107911
File: 1048576, Buffer: 268435456, Elapsed: 00:00:00.0099841
File: 1048576, Buffer: 536870912, Elapsed: 00:00:00.0135941
File: 1048576, Buffer: 1073741824, Elapsed: 00:00:00.0199248
File: 1073741824, Buffer: 1, Elapsed: 00:01:24.9154625
File: 1073741824, Buffer: 2, Elapsed: 00:01:13.0649384
File: 1073741824, Buffer: 4, Elapsed: 00:01:20.7892031
File: 1073741824, Buffer: 8, Elapsed: 00:01:14.3207983
File: 1073741824, Buffer: 16, Elapsed: 00:01:19.1491590
File: 1073741824, Buffer: 32, Elapsed: 00:01:13.1612342
File: 1073741824, Buffer: 64, Elapsed: 00:01:21.2668008
File: 1073741824, Buffer: 128, Elapsed: 00:01:12.1101598
File: 1073741824, Buffer: 256, Elapsed: 00:01:17.3565432
File: 1073741824, Buffer: 512, Elapsed: 00:01:14.2698611
File: 1073741824, Buffer: 1024, Elapsed: 00:01:28.4650820
File: 1073741824, Buffer: 2048, Elapsed: 00:00:20.3608743
File: 1073741824, Buffer: 4096, Elapsed: 00:00:13.4953693
File: 1073741824, Buffer: 8192, Elapsed: 00:00:09.7105073
File: 1073741824, Buffer: 16384, Elapsed: 00:00:07.8587447
File: 1073741824, Buffer: 32768, Elapsed: 00:00:06.9201750
File: 1073741824, Buffer: 65536, Elapsed: 00:00:06.4229520
File: 1073741824, Buffer: 131072, Elapsed: 00:00:05.9591664
File: 1073741824, Buffer: 262144, Elapsed: 00:00:05.7492599
File: 1073741824, Buffer: 524288, Elapsed: 00:00:05.6921116
File: 1073741824, Buffer: 1048576, Elapsed: 00:00:05.6765633
File: 1073741824, Buffer: 2097152, Elapsed: 00:00:05.7068171
File: 1073741824, Buffer: 4194304, Elapsed: 00:00:05.6758360
File: 1073741824, Buffer: 8388608, Elapsed: 00:00:05.5947740
File: 1073741824, Buffer: 16777216, Elapsed: 00:00:05.5885432
File: 1073741824, Buffer: 33554432, Elapsed: 00:00:06.0881748
File: 1073741824, Buffer: 67108864, Elapsed: 00:00:06.2771976
File: 1073741824, Buffer: 134217728, Elapsed: 00:00:06.2455592
File: 1073741824, Buffer: 268435456, Elapsed: 00:00:06.4810064
File: 1073741824, Buffer: 536870912, Elapsed: 00:00:06.4810521
File: 1073741824, Buffer: 1073741824, Elapsed: 00:00:06.5314156
Код для теста:
[TestClass]
public class WritePerfTests
{
[TestMethod]
public async Task FileWritePerformance()
{
var fileSize = 1L;
var useAsync = true;
using (Loggers.Register(ConsoleLogger.CreateDefault()))
using (var logger = Loggers.Register(FileLogger.CreateDefault(typeof(WritePerfTests))))
{
logger.ClearLog();
var bytes = Rng.Pseudo.GetBytes(1024);
for (var i = 0; i < 3; i++)
{
fileSize *= 1024;
for (var p = 0; p < 31; p++)
{
var buffer = (int)Math.Pow(2, p);
var file = new FileInfo($"d:/{fileSize}.{p}.test");
using (Disposable.Action(() => file.Delete()))
using (var stream = new FileStream(file.FullName, FileMode.Create, FileAccess.Write, FileShare.None, buffer, useAsync))
{
var w = Stopwatch.StartNew();
while (stream.Length < fileSize)
await stream.WriteAsync(bytes, 0, bytes.Length);
await stream.FlushAsync();
Loggers.Info($"File: {fileSize}, Buffer: {buffer}, Elapsed: {w.Elapsed}");
}
await Task.Delay(1000); //give file system chance to cleanup so next test wont be skewed
}
}
}
}
}