Question

我需要从文件中读取数据，处理并将结果写入另一个文件。我使用backgroundworker来显示进程状态。我写了这样的东西，用于后台工作的DoWork事件

private void ProcData(string fileToRead,string fileToWrite)
        {
            byte[] buffer = new byte[4 * 1024];

            //fileToRead & fileToWrite have same size
            FileInfo fileInfo = new FileInfo(fileToRead);

            using (FileStream streamReader = new FileStream(fileToRead, FileMode.Open))
            using (BinaryReader binaryReader = new BinaryReader(streamReader))
            using (FileStream streamWriter = new FileStream(fileToWrite, FileMode.Open))
            using (BinaryWriter binaryWriter = new BinaryWriter(streamWriter))
            {
                while (streamWriter.Position < fileInfo.Length)
                {
                    if (streamWriter.Position + buffer.Length > fileInfo.Length)
                    {
                        buffer = new byte[fileInfo.Length - streamWriter.Position];
                    }

                    //read
                    buffer = binaryReader.ReadBytes(buffer.Length);

                    //proccess
                    Proc(buffer);

                    //write
                    binaryWriter.Write(buffer);

                    //report if procentage changed
                    //...

                }//while
            }//using
        }

但它比从 fileToRead 读取并写入 fileToWrite 慢5倍，所以我想到了线程。我在网站上阅读了一些问题，并在this question

上尝试这样的问题

private void ProcData2(string fileToRead, string fileToWrite)
        {
            int threadNumber = 4; //for example

            Task[] tasks = new Task[threadNumber];

            long[] startByte = new long[threadNumber];
            long[] length = new long[threadNumber];

            //divide file to threadNumber(4) part
            //and update startByte & length

            var parentTask = Task.Run(() =>
            {
                for (int i = 0; i < threadNumber; i++)
                {
                    tasks[i] = Task.Factory.StartNew(() =>
                    {
                        Proc2(fileToRead, fileToWrite, startByte[i], length[i]);
                    });
                }
            });

            parentTask.Wait();

            Task.WaitAll(tasks);

        }
        //
        private void Proc2(string fileToRead,string fileToWrite,long fileStartByte,long partLength)
        { 
            byte[] buffer = new byte[4 * 1024];

            using (FileStream streamReader = new FileStream(fileToRead, FileMode.Open,FileAccess.Read,FileShare.Read))
            using (BinaryReader binaryReader = new BinaryReader(streamReader))
            using (FileStream streamWriter = new FileStream(fileToWrite, FileMode.Open,FileAccess.Write,FileShare.Write))
            using (BinaryWriter binaryWriter = new BinaryWriter(streamWriter))
            {
                streamReader.Seek(fileStartByte, SeekOrigin.Begin);
                streamWriter.Seek(fileStartByte, SeekOrigin.Begin);

                while (streamWriter.Position < fileStartByte+partLength)
                {
                    if (streamWriter.Position + buffer.Length > fileStartByte+partLength)
                    {
                        buffer = new byte[fileStartByte+partLength - streamWriter.Position];
                    }

                    //read
                    buffer = binaryReader.ReadBytes(buffer.Length);

                    //proccess
                    Proc(buffer);

                    //write
                    binaryWriter.Write(buffer);

                    //report if procentage changed
                    //...

                }//while
            }//using
        }

但我认为它有一些问题，每次切换任务都需要再次寻求。我考虑读取文件，使用线程进行Proc（）然后编写结果，但似乎错了。我该怎么做呢？（从文件中读取缓冲区，处理并使用任务将其写入其他文件）

// ============================================= ======================

基于Pete Kirkham的帖子我修改了我的方法。我不知道为什么，但它对我不起作用。我添加了新的方法，可以帮助他们。谢谢每个人的身体

 private void ProcData3(string fileToRead, string fileToWrite)
        {
            int bufferSize = 4 * 1024;
            int threadNumber = 4;//example
            List<byte[]> bufferPool = new List<byte[]>();
            Task[] tasks = new Task[threadNumber];

            //fileToRead & fileToWrite have same size
            FileInfo fileInfo = new FileInfo(fileToRead);

            using (FileStream streamReader = new FileStream(fileToRead, FileMode.Open))
            using (BinaryReader binaryReader = new BinaryReader(streamReader))
            using (FileStream streamWriter = new FileStream(fileToWrite, FileMode.Open))
            using (BinaryWriter binaryWriter = new BinaryWriter(streamWriter))
            {
                while (streamWriter.Position < fileInfo.Length)
                {
                    //read
                    for (int g = 0; g < threadNumber; g++)
                    {
                        if (streamWriter.Position + bufferSize <= fileInfo.Length)
                        {
                            bufferPool.Add(binaryReader.ReadBytes(bufferSize));
                        }
                        else
                        {
                            bufferPool.Add(binaryReader.ReadBytes((int)(fileInfo.Length - streamWriter.Position)));
                            break;
                        }
                    }

                    //do
                    var parentTask = Task.Run(() =>
                    {
                        for (int th = 0; th < bufferPool.Count; th++)
                        {
                            int index = th;

                            //threads
                            tasks[index] = Task.Factory.StartNew(() =>
                            {
                                Proc(bufferPool[index]);
                            });

                        }//for th
                    });

                    //stop parent task(run childs)
                    parentTask.Wait();

                    //wait till all task be done
                    Task.WaitAll(tasks);

                    //write
                    for (int g = 0; g < bufferPool.Count; g++)
                    {
                        binaryWriter.Write(bufferPool[g]);
                    }

                    //report if procentage changed
                    //...

                }//while
            }//using
        }

Answer 1

基本上，您希望将数据处理拆分为并行任务，但您不希望想要拆分IO。

这种情况如何发生取决于数据的大小。如果它足够小以适应内存，那么你可以将它全部读入一个输入数组并创建一个输出数组，然后创建任务来处理一些输入数组并填充一些输出数组，然后编写整个输出数组提交。

如果数据太大，那么您需要限制一次读取和写入的数据量。因此，您的主要流程通过读取N个数据块并创建N个任务来处理它们。然后等待任务按顺序完成，每次完成后，编写输出块并读取新的输入块并创建另一个任务。对于N和块大小的良好值，需要进行一些实验，这意味着任务往往以与IO工作时相同的速率完成。

使用线程（任务）进行工作包含I / O.

1 个答案: