我是使用mysql进行多线程处理的新手。我正在实现一个程序,该程序读取包含出租车行程数据的大型csv文件,并在与csv文件具有相同字段的表中插入行。 由于csv文件包含大量的行(600K),我正在考虑使用单线连接进行多线程插入,但是我无法处理这个问题。似乎插入没有被提交或者像这样。我应该提一下,当我不使用多线程时,一切都很好。
以下是代码的一部分:
TestHibernateInsert.java:
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.hibernate.Session;
public class TestHibernateInsert {
public static void main(String[] args) throws IOException
{
BufferedReader reader ;
String line;
String[] fields;
Session session = HibernateUtil.getSessionFactory().openSession();
Path inputPath = new Path("/home/rafik/HADOOP/TUTORIALS/test/assignmentHDP/input/yellow-taxi.csv");
NYTaxi taxiTrip;
Configuration conf = new Configuration();
FileSystem fs = inputPath.getFileSystem(conf);
FSDataInputStream inputStream = fs.open(inputPath);
ExecutorService executor = Executors.newFixedThreadPool(10);
reader = new BufferedReader(new InputStreamReader(inputStream));
reader.readLine();
while ((line = reader.readLine()) != null && (fields=line.split(",")).length == 19) {
taxiTrip = new NYTaxi();
Runnable insert = new InsertThread(fields, taxiTrip, session);
executor.execute(insert);
// insertLine(fields, taxiTrip, session);
}
executor.shutdown();
while (!executor.isTerminated()) { }
HibernateUtil.shutdown();
}
}
HibernateUtil.java:
import java.io.File;
import org.hibernate.SessionFactory;
import org.hibernate.cfg.AnnotationConfiguration;
public class HibernateUtil
{
private static final SessionFactory sessionFactory = buildSessionFactory();
private static SessionFactory buildSessionFactory()
{
try
{
// Create the SessionFactory from hibernate.cfg.xml
return new AnnotationConfiguration().configure(new File("resources/hibernate.cgf.xml")).buildSessionFactory();
}
catch (Throwable ex) {
// Make sure you log the exception, as it might be swallowed
System.err.println("Initial SessionFactory creation failed. " + ex);
throw new ExceptionInInitializerError(ex);
}
}
public static SessionFactory getSessionFactory() {
return sessionFactory;
}
public static void shutdown() {
// Close caches and connection pools
getSessionFactory().close();
}
}
InsertThread.java:
import java.sql.Timestamp;
import org.hibernate.Session;
public class InsertThread implements Runnable {
public String[] fields;
public NYTaxi taxiTrip;
public Session session;
public InsertThread(String[] fields, NYTaxi taxiTrip, Session session) {
this.fields = fields;
this.taxiTrip = taxiTrip;
this.session = session;
}
@Override
public void run() {
System.out.println(Thread.currentThread().getName()+" (Start) ");
insertLine(fields, taxiTrip, session);
try {
Thread.sleep(2000);
} catch (InterruptedException e) {}
System.out.println(Thread.currentThread().getName()+" (END) ");
}
public static void insertLine(String[] fields, NYTaxi taxiTrip, Session session) {
try {
taxiTrip.setTripID(Long.parseLong(fields[0]));
taxiTrip.setDropoffDatetime(new Timestamp(Long.parseLong(fields[1])));
taxiTrip.setDropoffLatitude(Double.parseDouble(fields[2]));
taxiTrip.setDropoffLongitude(Double.parseDouble(fields[3]));
taxiTrip.setDropoffTaxizoneId(Integer.parseInt(fields[4]));
taxiTrip.setExtra(Float.parseFloat(fields[5]));
taxiTrip.setFareAmount(Double.parseDouble(fields[6]));
taxiTrip.setPaymentType(fields[7]);
taxiTrip.setPickupDatetime(new Timestamp(Long.parseLong(fields[8])));
taxiTrip.setPickupLatitude(Double.parseDouble(fields[9]));
taxiTrip.setPickupLongitude(Double.parseDouble(fields[10]));
taxiTrip.setPickupTaxizoneId(Integer.parseInt(fields[11]));
taxiTrip.setStoreAndFwdFlag(fields[12]);
taxiTrip.setTipAmount(Float.parseFloat(fields[13]));
taxiTrip.setTollsAmount(Float.parseFloat(fields[14]));
taxiTrip.setTotalAmount(Double.parseDouble(fields[15]));
taxiTrip.setTripDistance(Float.parseFloat(fields[16]));
taxiTrip.setTripType(fields[17]);
taxiTrip.setVendorId(fields[18]);
} catch (Exception e) {}
//begin the insertion transaction
session.beginTransaction();
//Save the trip in database
session.save(taxiTrip);
// if (!session.getTransaction().wasCommitted()) {
//Commit the transaction
session.getTransaction().commit();
// }
}
}
当我只是将insertLine(fields, taxiTrip, session);
放在main方法中并注释多线程执行时,一切正常并且表正在更新,但是使用多线程时,表不会更新!
任何帮助表示赞赏!! Tahnks:)