我有一个大型文件中的以下代码,比如超过一百万行。我正在使用Parallel和Linq方法。有没有更好的方法呢?如果是,那怎么办?
private static void ReadFile()
{
float floatTester = 0;
List<float[]> result = File.ReadLines(@"largedata.csv")
.Where(l => !string.IsNullOrWhiteSpace(l))
.Select(l => new { Line = l, Fields = l.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries) })
.Select(x => x.Fields
.Where(f => Single.TryParse(f, out floatTester))
.Select(f => floatTester).ToArray())
.ToList();
// now get your totals
int numberOfLinesWithData = result.Count;
int numberOfAllFloats = result.Sum(fa => fa.Length);
MessageBox.Show(numberOfAllFloats.ToString());
}
private static readonly char[] Separators = { ',', ' ' };
private static void ProcessFile()
{
var lines = File.ReadAllLines("largedata.csv");
var numbers = ProcessRawNumbers(lines);
var rowTotal = new List<double>();
var totalElements = 0;
foreach (var values in numbers)
{
var sumOfRow = values.Sum();
rowTotal.Add(sumOfRow);
totalElements += values.Count;
}
MessageBox.Show(totalElements.ToString());
}
private static List<List<double>> ProcessRawNumbers(IEnumerable<string> lines)
{
var numbers = new List<List<double>>();
/*System.Threading.Tasks.*/
Parallel.ForEach(lines, line =>
{
lock (numbers)
{
numbers.Add(ProcessLine(line));
}
});
return numbers;
}
private static List<double> ProcessLine(string line)
{
var list = new List<double>();
foreach (var s in line.Split(Separators, StringSplitOptions.RemoveEmptyEntries))
{
double i;
if (Double.TryParse(s, out i))
{
list.Add(i);
}
}
return list;
}
private void button1_Click(object sender, EventArgs e)
{
Stopwatch stopWatchParallel = new Stopwatch();
stopWatchParallel.Start();
ProcessFile();
stopWatchParallel.Stop();
// Get the elapsed time as a TimeSpan value.
TimeSpan ts = stopWatchParallel.Elapsed;
// Format and display the TimeSpan value.
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds,
ts.Milliseconds / 10);
MessageBox.Show(elapsedTime);
Stopwatch stopWatchLinQ = new Stopwatch();
stopWatchLinQ.Start();
ReadFile();
stopWatchLinQ.Stop();
// Get the elapsed time as a TimeSpan value.
TimeSpan ts2 = stopWatchLinQ.Elapsed;
// Format and display the TimeSpan value.
string elapsedTimeLinQ = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts2.Hours, ts.Minutes, ts.Seconds,
ts2.Milliseconds / 10);
MessageBox.Show(elapsedTimeLinQ);
}
答案 0 :(得分:2)
你可以使用内置的OleDb ..
public void ImportCsvFile(string filename)
{
FileInfo file = new FileInfo(filename);
using (OleDbConnection con =
new OleDbConnection("Provider=Microsoft.Jet.OLEDB.4.0;Data Source=\"" +
file.DirectoryName + "\";
Extended Properties='text;HDR=Yes;FMT=Delimited(,)';"))
{
using (OleDbCommand cmd = new OleDbCommand(string.Format
("SELECT * FROM [{0}]", file.Name), con))
{
con.Open();
// Using a DataTable to process the data
using (OleDbDataAdapter adp = new OleDbDataAdapter(cmd))
{
DataTable tbl = new DataTable("MyTable");
adp.Fill(tbl);
//foreach (DataRow row in tbl.Rows)
//Or directly make a list
List<DataRow> list = dt.AsEnumerable().ToList();
}
}
}
}
答案 1 :(得分:2)
答案 2 :(得分:1)
最近我遇到了为同一目的尽可能快地解析大型CSV文件的问题:数据聚合和指标计算(在我的情况下,最终目标是生成数据透视表)。我测试了大多数流行的CSV阅读器,但发现它们不是为解析具有数百万行或更多行的CSV文件而设计的; JoshClose的CsvHelper很快,但最后我能够以2x-4x倍的速度处理CSV作为流!
我的方法基于2个假设:
foreach
) - 因为for
效率更高。实际使用数字(数据透视表由200MB CSV文件,17列,只有3列用于构建交叉表):
---更新---
我已经发布了我的库,其工作原理如上所述:https://github.com/nreco/csv
答案 3 :(得分:0)
你应该看看CsvHelper =&gt; https://github.com/JoshClose/CsvHelper/
它允许您将.csv文件映射到类,因此您可以将.csv文件用作对象。尝试一下,然后尝试应用并行操作,看看你是否有更好的性能。
以下是我对项目的示例代码:
using (var csv = new CsvReader(new StreamReader(filePath, Encoding.Default)))
{
csv.Configuration.Delimiter = ';';
csv.Configuration.ClassMapping<LogHeaderMap, LogHeader>();
var data = csv.GetRecords<LogHeader>();
foreach (var entry in data.OrderByDescending(x => x.Date))
{
//process
}
}