MySQL多线程插入单个数据库连接无法正常工作

时间:2018-03-21 13:23:20

标签: mysql multithreading

我是使用mysql进行多线程处理的新手。我正在实现一个程序,该程序读取包含出租车行程数据的大型csv文件,并在与csv文件具有相同字段的表中插入行。 由于csv文件包含大量的行(600K),我正在考虑使用单线连接进行多线程插入,但是我无法处理这个问题。似乎插入没有被提交或者像这样。我应该提一下,当我不使用多线程时,一切都很好。

以下是代码的一部分:

TestHibernateInsert.java:

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.hibernate.Session;

public class TestHibernateInsert {

    public static void main(String[] args) throws IOException
    {
        BufferedReader reader ;
        String line;
        String[] fields;
        Session session = HibernateUtil.getSessionFactory().openSession();
        Path inputPath = new Path("/home/rafik/HADOOP/TUTORIALS/test/assignmentHDP/input/yellow-taxi.csv");
        NYTaxi taxiTrip;
        Configuration conf = new Configuration();
        FileSystem fs = inputPath.getFileSystem(conf);
        FSDataInputStream inputStream = fs.open(inputPath);
        ExecutorService executor = Executors.newFixedThreadPool(10);
        reader = new BufferedReader(new InputStreamReader(inputStream));
        reader.readLine();
        while ((line = reader.readLine()) != null && (fields=line.split(",")).length == 19) {
            taxiTrip = new NYTaxi();
            Runnable insert = new InsertThread(fields, taxiTrip, session);
            executor.execute(insert);
//          insertLine(fields, taxiTrip, session);
        }
        executor.shutdown();  
        while (!executor.isTerminated()) {   }  
        HibernateUtil.shutdown();
    }
}

HibernateUtil.java:

import java.io.File;
import org.hibernate.SessionFactory;
import org.hibernate.cfg.AnnotationConfiguration;

public class HibernateUtil
{
    private static final SessionFactory sessionFactory = buildSessionFactory();

    private static SessionFactory buildSessionFactory()
    {
        try
        {
            // Create the SessionFactory from hibernate.cfg.xml
            return new AnnotationConfiguration().configure(new File("resources/hibernate.cgf.xml")).buildSessionFactory();
        }
        catch (Throwable ex) {
            // Make sure you log the exception, as it might be swallowed
            System.err.println("Initial SessionFactory creation failed. " + ex);
            throw new ExceptionInInitializerError(ex);
        }
    }

    public static SessionFactory getSessionFactory() {
        return sessionFactory;
    }

    public static void shutdown() {
        // Close caches and connection pools
        getSessionFactory().close();
    }
}

InsertThread.java:

import java.sql.Timestamp;

import org.hibernate.Session;

public class InsertThread implements Runnable {
        public String[] fields;
        public NYTaxi       taxiTrip;
        public Session      session;

        public InsertThread(String[] fields, NYTaxi taxiTrip, Session session) {
            this.fields = fields;
            this.taxiTrip = taxiTrip;
            this.session = session;
        }
        @Override
        public void run() {
            System.out.println(Thread.currentThread().getName()+" (Start) ");  
            insertLine(fields, taxiTrip, session);
            try {
                Thread.sleep(2000);
            } catch (InterruptedException e) {}
            System.out.println(Thread.currentThread().getName()+" (END) "); 
        }

        public static void insertLine(String[] fields, NYTaxi taxiTrip, Session session) {
            try {
                taxiTrip.setTripID(Long.parseLong(fields[0]));
                taxiTrip.setDropoffDatetime(new Timestamp(Long.parseLong(fields[1])));
                taxiTrip.setDropoffLatitude(Double.parseDouble(fields[2]));
                taxiTrip.setDropoffLongitude(Double.parseDouble(fields[3]));
                taxiTrip.setDropoffTaxizoneId(Integer.parseInt(fields[4]));
                taxiTrip.setExtra(Float.parseFloat(fields[5]));
                taxiTrip.setFareAmount(Double.parseDouble(fields[6]));
                taxiTrip.setPaymentType(fields[7]);
                taxiTrip.setPickupDatetime(new Timestamp(Long.parseLong(fields[8])));
                taxiTrip.setPickupLatitude(Double.parseDouble(fields[9]));
                taxiTrip.setPickupLongitude(Double.parseDouble(fields[10]));
                taxiTrip.setPickupTaxizoneId(Integer.parseInt(fields[11]));
                taxiTrip.setStoreAndFwdFlag(fields[12]);
                taxiTrip.setTipAmount(Float.parseFloat(fields[13]));
                taxiTrip.setTollsAmount(Float.parseFloat(fields[14]));
                taxiTrip.setTotalAmount(Double.parseDouble(fields[15]));
                taxiTrip.setTripDistance(Float.parseFloat(fields[16]));
                taxiTrip.setTripType(fields[17]);
                taxiTrip.setVendorId(fields[18]);
            } catch (Exception e) {}
            //begin the insertion transaction
            session.beginTransaction();


            //Save the trip in database
            session.save(taxiTrip);

//          if (!session.getTransaction().wasCommitted()) {
                //Commit the transaction
                session.getTransaction().commit();
//          }
        }
    }

当我只是将insertLine(fields, taxiTrip, session);放在main方法中并注释多线程执行时,一切正常并且表正在更新,但是使用多线程时,表不会更新!

任何帮助表示赞赏!! Tahnks:)

0 个答案:

没有答案