我有一个巨大的CSV文件~800 K记录并使用该数据我必须形成一个POST请求并进行休息呼叫。 最初我尝试不使用线程但是处理它需要很长时间。
所以我想到使用线程来加速这个过程并遵循以下方法。
上述两种方法都需要太长时间才能完成一半（> 3小时）。可能其中一个不是好方法。
有人可以提供建议，帮助我加快这个过程吗？使用线程不是强制性的。
只是添加我的代码在消费者端并且对生产者端没有太多控制权。
MultiThreadedFileRead（扩展线程的主类）
class MultiThreadedFileRead extends Thread
{
public static final String EMPTY = "";
public static Logger logger = Logger.getLogger(MultiThreadedFileRead.class);
BufferedReader bReader = null;
String filename = null;
int Request_Num = 0;
HashMap<Integer, String> filteredSrcDataMap = null;
List<BrandRQ> brandRQList = null;
List<BrandRS> brandRSList = null;
ReadDLShipFile readDLShipFile = new ReadDLShipFile();
GenerateAndProcessPostBrandAPI generateAndProcessPostBrandAPI = new GenerateAndProcessPostBrandAPI();
MultiThreadedFileRead()
{
}
MultiThreadedFileRead(String fname) throws Exception
{
filename = fname;
Request_Num = 0;
List<BrandRQ> brandRQList = new ArrayList<BrandRQ>();
List<BrandRS> brandRSList = new ArrayList<BrandRS>();
this.start();
}
public void run()
{
logger.debug("File *** : " + this.filename);
//Generate Source Data Map
HashMap<Integer, String> filteredSrcDataMap = (HashMap<Integer, String>) readDLShipFile.readSourceFileData(this.filename);
generateAndProcessPostBrandAPI.generateAndProcessPostRequest(filteredSrcDataMap, this);
}
public static void main(String args[]) throws Exception
{
String environment = ""; // TO BE UNCOMMENTED
String outPutfileName = ""; // TO BE UNCOMMENTED
BrandRetrivalProperties brProps = BrandRetrivalProperties.getInstance();
if(args != null && args.length == 2 && DLPremiumUtil.isNotEmpty(args[0]) && DLPremiumUtil.isNotEmpty(args[1])) // TO BE UNCOMMENTED
{
environment = args[0].trim();
outPutfileName = args[1].trim();
ApplicationInitializer applicationInitializer = new ApplicationInitializer(environment);
applicationInitializer.initializeParams();
logger.debug("Environment : " + environment);
logger.debug("Filename : " + outPutfileName);
// TO BE UNCOMMENTED - START
}else{
System.out.println("Invalid parameters passed. Expected paramenters: Environment");
System.out.println("Sample Request: java -jar BrandRetrival.jar [prd|int] brand.dat");
System.exit(1);
}
MultiThreadedFileRead multiThreadedFileRead = new MultiThreadedFileRead();
List<String> listOfFileNames = multiThreadedFileRead.getFileNames(brProps);
logger.debug("List : " + listOfFileNames);
int totalFileSize = listOfFileNames.size();
logger.debug("totalFileSize : " + totalFileSize);
MultiThreadedFileRead fr[]=new MultiThreadedFileRead[totalFileSize];
logger.debug("Size of Array : " + fr.length);
for(int i = 0; i < totalFileSize; i++)
{
logger.debug("\nFile Getting Added to Array : " + brProps.getCountFilePath()+listOfFileNames.get(i));
fr[i]=new MultiThreadedFileRead(brProps.getCountFilePath()+listOfFileNames.get(i));
}
}
public List<String> getFileNames(BrandRetrivalProperties brProps)
{
MultiThreadedFileRead multiThreadedFileRead1 = new MultiThreadedFileRead();
BufferedReader br = null;
String line = "";
String filename = multiThreadedFileRead1.getCounterFileName(brProps.getCountFilePath(), brProps.getCountFilePathStartsWith());
List<String> fileNameList = null;
logger.debug("filename : " + filename);
if(filename != null)
{
fileNameList = new ArrayList<String>();
try{
br = new BufferedReader(new FileReader(brProps.getCountFilePath()+filename));
while ((line = br.readLine()) != null)
{
fileNameList.add(line);
}
} catch (FileNotFoundException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
} catch (IOException e2) {
// TODO Auto-generated catch block ;
e2.printStackTrace();
} catch (Exception e3) {
// TODO Auto-generated catch block
e3.printStackTrace();
} finally
{
if (br != null)
{
try
{
br.close();
} catch (IOException e4)
{
e4.printStackTrace();
} catch (Exception e5)
{
e5.printStackTrace();
}
}
}
}
return fileNameList;
}
// Get the Name of the file which has name of individual CSV files to read to form the request
public String getCounterFileName(String sourceFilePath, String sourceFileNameStartsWith)
{
String fileName = null;
if(sourceFilePath != null && !sourceFilePath.equals(EMPTY) &&
sourceFileNameStartsWith != null && !sourceFileNameStartsWith.equals(EMPTY))
{
File filePath = new File(sourceFilePath);
File[] listOfFiles = filePath.listFiles();
for (int i = 0; i < listOfFiles.length; i++) {
if (listOfFiles[i].isFile() && listOfFiles[i].getName().startsWith(sourceFileNameStartsWith)) {
fileName = listOfFiles[i].getName();
} else if (listOfFiles[i].isDirectory()) {
fileName = null;
}
}
}
return fileName;
}
}
GenerateAndProcessPostBrandAPI
public class GenerateAndProcessPostBrandAPI {
public static Logger logger = Logger.getLogger(GenerateAndProcessPostBrandAPI.class);
List<BrandRQ> brandRQList = new ArrayList<BrandRQ>();
List<BrandRS> brandRSList = new ArrayList<BrandRS>();
DLPremiumClientPost dlPremiumclientPost = new DLPremiumClientPost();
JSONRequestGenerator jsonRequestGenerator = new JSONRequestGenerator();
public List<BrandRQ> getBrandRequstList(Map<Integer, String> filteredSrcDataMap)
{
if(filteredSrcDataMap != null)
{
brandRQList = jsonRequestGenerator.getBrandRQList(filteredSrcDataMap, brandRQList);
}
return brandRQList;
}
public void generateAndProcessPostRequest(Map<Integer, String> filteredSrcDataMap, MultiThreadedFileRead multiThreadedFileRead)
{
if(filteredSrcDataMap != null)
{
brandRQList = jsonRequestGenerator.getBrandRQList(filteredSrcDataMap, brandRQList);
if(brandRQList != null && brandRQList.size() > 0)
{
BrandRetrivalProperties brProps = BrandRetrivalProperties.getInstance();
String postBrandsEndpoint = brProps.getPostBrandsEndPoint();
for (BrandRQ brandRQ : brandRQList)
{
multiThreadedFileRead.Request_Num = multiThreadedFileRead.Request_Num + 1;
logger.debug("Brand Request - " + multiThreadedFileRead.Request_Num + " : " + ObjectToJSONCovertor.converToJSON(brandRQ));
BrandRS brandResponse = dlPremiumclientPost.loadBrandApiResponseJSON(brandRQ, postBrandsEndpoint, DLPremiumUtil.getTransactionID(), BrandConstants.AUTH_TOKEN);
logger.debug("Response - " + multiThreadedFileRead.Request_Num + " : " + ObjectToJSONCovertor.converToJSON(brandResponse));
brandRSList.add(brandResponse);
}
}
}
}
}