我使用下面的代码来处理大的CSV文件，它将为每2000条记录拆分CSV并安排一个线程来处理2000条记录，最后，在所有线程完成执行后，将采取未来。
我遇到以下问题， 当我有2000条记录的CSV：1个线程需要32秒 但是CSV 0f 8000记录：4个线程大约需要1.3分钟。在这里，我可以看到所有线程同时启动并同时完成执行。
这是什么问题？据我们在处理线程时可以理解，当我们增加线程数时，每个线程所花费的时间应该是相同的。但就我而言，如果我增加线程数，则所花费的时间会增加
public void parseCSVandUpload(InputStream inputStream) throws ExecutionException, InterruptedException, IOException
{
List<Future<String>> results = new ArrayList<Future<String>>();
CSVReader reader = null;
try
{
List<String[]> csvPart = new ArrayList<String[]>();
reader = new CSVReader(new InputStreamReader(inputStream));
String[] nextLine;
int counter = 0;
ExecutorService csvScheduler = Executors.newFixedThreadPool(10);
while((nextLine = reader.readNext()) != null)
{
csvPart.add(nextLine);
counter++;
if(counter == 2000)
{
count++;
Future<String> result = csvScheduler.submit(new uploadCSV(csvPart));
results.add(result);
csvPart = new ArrayList<String[]>();
counter = 0;
}
}
count++;
if (csvPart.size() > 0)
{
Future<String> result = csvScheduler.submit(new uploadCSV(csvPart));
results.add(result);
}
}
catch (Exception e)
{
throw e;
}
finally
{
try
{
if(reader!=null)
{
reader.close();
}
}
catch (IOException ioe)
{
}
}
for(Future<String> result: results )
{
System.out.println(result.get());
}
}
class uploadCSV implements Callable<BulkResult>
{
protected final Logger logger = LoggerFactory.getLogger(uploadCSV.class);
private List<String[]> rows ;
public uploadCSV(List<String[]> rows)
{
this.rows = rows;
}
private List<Map<String,Object>> parseColumn5(String profileName, final String[] nextLine) throws RepositoryException
{
List<String> lines = new ArrayList<String>();
for(int column = 4; column <nextLine.length; column++)
{
String lineInfo = this.parseLine(nextLine, column);
if(column == 4 && Util.isNullOrEmpty(lineInfo))
{
return lines;
}
if(!Util.isNullOrEmpty(lineInfo))
{
lines.add(lineInfo);
}
}
return lines;
}
private String parseColumn1(final String[] csvRow)
{
String column = null;
if (csvRow != null && csvRow.length>0)
{
column = csvRow[0];
}
return column;
}
private String parseColumn2(final String[] csvRow)
{
String column = null;
if (csvRow != null && csvRow.length>1)
{
column = csvRow[1];
}
return column;
}
private String parseColumn3(final String[] csvRow)
{
String column = null;
if (csvRow != null && csvRow.length>2)
{
column = csvRow[2];
}
return column;
}
private String parseColumn4(final String[] csvRow)
{
String column = null;
if (csvRow != null && csvRow.length>3)
{
column = csvRow[3];
}
return column;
}
private String parseLine(final String[] csvRow, final int column)
{
String dnInfo = null;
if (csvRow != null && csvRow.length>column)
{
dnInfo = csvRow[column];
}
return dnInfo;
}
public String call() throws IOException, RepositoryException
{
String result = "Error";
try
{
for(String[] nextLine : rows)
{
try
{
String macAddress = this.parseColumn1(nextLine);
if (macAddress != null)
{
String vendor = this.parseColumn2(nextLine);
if(vendor == null)
{
continue;
}
//Get model
String model = this.parseColumn3(nextLine);
if(model == null)
{
continue;
}
//Get profileName
String profileName = this.parseColumn4(nextLine);
if(profileName == null)
{
continue;
}
//Get the extensions
List<Map<String,Object>> lines = this.parseColumn5(profileName, nextLine);
//Error if the mandatory extension is not provided
if (lines.isEmpty())
{
//error
return
}
else // process only if there is a extension
{
deviceParams.put(Device.MAC_ADDRESS, macAddress);
deviceParams.put(Device.VENDOR, vendor);
deviceParams.put(Device.MODEL, model);
deviceParams.put(Device.PROFILE_NAME, profileName);
ReturnCode responseCode = handleOperation(device, Device.UPDATE_DEVICE, deviceParams);
this.addException(responseCode, macAddress, Device.UPDATE_DEVICE, ret,true);
if(responseCode == ReturnCode.Ok)
{
result = "Success";
}
}
}
else
{
this.logger.error("[DM-Bulk] Cannot process empty row:{}",StringUtils.join(nextLine, ","));
}
catch (Exception e)
{
this.logger.error(String.format("[DM-Bulk] Cannot process input row %s: %s", StringUtils.join(nextLine, ","), e.getMessage()));
}
}
}
catch (Exception e)
{
throw e;
}
finally
{
try
{
if(reader!=null)
{
reader.close();
}
}
catch (IOException ioe)
{
}
}
return result;
}
}