Question

我创建了一个数据读取器流，我正在尝试将结果写入文件。由于此表可能会返回数百万条记录，因此我想写入多个文件，以便我可以在文本编辑器中打开它们而不会出现问题，即。文本编辑器崩溃，因为文件太大。这大致是我现在所拥有的。

using (var connection = new SqlConnection(connectionString))
using (var stream = new FileStream("directoryLocation", FileMode.Create))
{
    SqlCommand command = connection.CreateCommand();
    command.CommandText = "Select * from tblTemp";
    connection.Open();

    using(SqlDataReader reader = command.ExecuteReader())
    {
        var tableName = "tblTemp";
        var fileName = tableName + ".txt";
        var recordCount = 0;
        var fileCount = 0;

        using (StreamWriter writer = new StreamWriter(stream.Open()))
        {
            while(reader.Read())
            {
                if(recordCount == 500000)
                {
                    // Right here. Need to figure out how to close old file start new
                    recordCount = 0;
                    writer.Close();

                    fileName = tableName + "_" + (++fileCount).ToString() + ".txt";
                    writer = new StreamWriter(fileName); // I know this doesn't work. Just sudo code
                }

                recordCount++;
                writer.WriterLine(recordInfo); // recordInfo is sudo code as well
            }
        }
    }
}

我不想将using writer语句移动到reader循环中，因为这会打开和关闭每个记录的文件连接。有关如何保持我在阅读器中的位置以及仅在需要时打开和关闭文件的任何想法？

Answer 1

你很亲密。您不需要单独的FileStream。只要根据需要处理前一个StreamWriter，就可以在循环中创建一个新的StreamWriter。

using (var connection = new SqlConnection(connectionString))
{
    SqlCommand command = connection.CreateCommand();
    command.CommandText = "Select * from tblTemp";
    connection.Open();

    using(SqlDataReader reader = command.ExecuteReader())
    {
        var tableName = "tblTemp";
        var fileName = tableName + ".txt";
        var recordCount = 0;
        var fileCount = 0;

        StreamWriter writer = null;
        try
        {
            while (reader.Read())
            {
                if (writer == null || recordCount == 500000)
                {
                    recordCount = 0;

                    // Close the previous file if it is open...
                    if (writer != null)
                    {
                        writer.Close();
                        writer.Dispose();
                    }

                    fileName = tableName + "_" + (++fileCount).ToString() + ".txt";

                    // Open the new file...
                    writer = new StreamWriter(fileName);
                }

                recordCount++;
                writer.WriterLine(recordInfo); // recordInfo is sudo code as well
            }
        }
        finally
        {
            // Make sure the file gets closed...
            if (writer != null)
            {
                writer.Dispose();
            }
        }
    }
}

Answer 2

就个人而言，我会使用TextWriter的自定义覆盖来解除使用代码滚动文件名的逻辑。这样的东西完全没有经过测试，可能不是线程安全的，效率低下的（它目前只是一次写一个字符！），并且可能有错误的代码：

public class RollingFileWriter : TextWriter
{
    private readonly string _filenamePrefix;
    private readonly string _fileNameSuffix;
    private readonly int _maxRecordCount;
    private Stream _innerStream;
    private int _recordCount = 0;
    private int _fileCounter = 0;

    public RollingFileWriter( string filenamePrefix, string fileNameSuffix = ".txt", int maxRecordCount = 500000 )
    {
        _filenamePrefix = filenamePrefix;
        _fileNameSuffix = fileNameSuffix;
        _maxRecordCount = maxRecordCount;

        _innerStream = new FileStream(
            _filenamePrefix + "_" + _fileCounter.ToString() + _fileNameSuffix,
            FileMode.Create );
    }

    public override Encoding Encoding
    {
        get { return Encoding.UTF8; }
    }

    public override void Write( char value )
    {
        _innerStream.Write( Encoding.GetBytes( new[] { value } ), 0, 1 );
    }

    public override void WriteLine( string value )
    {
        if ( ++_recordCount == _maxRecordCount )
        {
            SwitchStreams();
        }
        base.WriteLine( value );
    }

    private void SwitchStreams()
    {
        _innerStream.Close();
        _innerStream.Dispose();

        _innerStream = new FileStream(
            _filenamePrefix + "_" + ( ++_fileCounter ).ToString() + _fileNameSuffix,
            FileMode.Create );
        _recordCount = 0;
    }

    protected override void Dispose( bool disposing )
    {
        if ( disposing )
        {
            _innerStream.Dispose();
        }
    }
}

然后你可以摆脱外部FileStream，并用StreamWriter替换内部RollingFileWriter，并从循环中删除所有其他逻辑。

从SqlDataReader写入多个文件

2 个答案: