提问人:Usman 提问时间:9/17/2023 更新时间:9/17/2023 访问量:29
使用内存映射 IO 文件时遇到过多的内存不足异常
Facing too much Insufficient Memory exception while working with memory mapped-IO files
问:
我正在尝试创建内存映射的 IO 文件,以调整大小 (1-100 GB) 文件。这些文件将包含在此过程中生成的随机数据。
因此,我尝试通过考虑系统的可用内存来动态生成分区。
以下是源代码。
public class RandomLinesGenerator
{
private static string GenerateRandomTextLine(int minValue, int maxValue)
{
const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
Random random = new Random();
int length = random.Next(minValue, maxValue);
return new string(Enumerable.Repeat(chars, length)
.Select(s => s[random.Next(s.Length)]).ToArray());
}
public static byte[] GetRandomBytes(long desiredByteCount, int prefixIntMinValue, int prefixIntMaxValue)
{
Random random = new Random();
long currentByteCount = 0;
StringBuilder builder = new StringBuilder();
while (currentByteCount < desiredByteCount)
{
// Generate a random integer
int randomNumber = random.Next(prefixIntMinValue, prefixIntMaxValue + 1);
// Generate a random line
string randomLine = $"{randomNumber} {GenerateRandomTextLine(1, 150)}";
// Calculate the byte count of the line
long lineByteCount = Encoding.UTF8.GetByteCount(randomLine) + Environment.NewLine.Length;
// If adding the line exceeds the desired byte count, break the loop
if (currentByteCount + lineByteCount > desiredByteCount)
{
break;
}
builder.AppendLine(randomLine); // ==> This is the line where the exception is occurring of Insufficient memory..
currentByteCount += lineByteCount;
}
var bytes = Encoding.UTF8.GetBytes(builder.ToString());
builder.Clear();
return bytes;
}
}
public class MemoryMappedIOTechnique
{
private static readonly string _fileLocation = Path.Combine("Resources", "files");
private long GetPartitionSize(long totalFileSize)
{
var partitionSize = (long)Math.Ceiling((long)new PerformanceCounter("Memory", "Available Bytes").NextValue() * 0.8);
return Math.Min(partitionSize ,totalFileSize);
}
public void Execute(int sizeInGb)
{
try
{
long totalBytesToWrite = sizeInGb * Convert.ToInt64(1024 * 1024 * 1024);
string filePath = Path.Combine(_fileLocation, $"file_{sizeInGb}-GB.txt");
long partitionSize = GetPartitionSize(totalBytesToWrite);
using (MemoryMappedFile mmf = MemoryMappedFile.CreateFromFile(filePath, FileMode.Create, "mmf", totalBytesToWrite))
{
long offset = 0;
long partitions = (totalBytesToWrite / partitionSize);
for (var partition = 0; partition < partitions; partition++)
{
using (var accessor = mmf.CreateViewAccessor(offset, partitionSize))
{
var randomBytes = RandomLinesGenerator.GetRandomBytes(partitionSize, 10, 2500);
accessor.WriteArray(0, randomBytes, 0, randomBytes.Length);
offset += partitionSize;
}
}
}
}
catch(Exception ex)
{
}
}
现在,当我刚刚调用提供 2 GB 的函数时,我遇到了异常。首先,它花费了更多的时间,另一个是导致“内存不足”的异常 我可以看到在任务管理器上,几乎 3 GB 仍然可用。
谁能指导我如何更改源代码以安全地处理所有类型的情况,而不会出现内存不足的异常情况?需要生成 ( 1, 2, ...-100 GB ) 文件。
我对内存映射 IO 完全陌生,对整个主题知之甚少。
答: 暂无答案
评论