Question

您好我面临的问题是在具有两列（Id＆amp; Id）的表中插入1lac数据。任何人都可以优化代码。

public class edgeConnection {
static ArrayList al3 = new ArrayList();
static HashSet set=null;
//static HashMap hm = null;
//static int val ;
//Database connection
public static DataSource getMySQLDataSource() throws Exception {
    Properties props = new Properties();
    FileInputStream fis = null;
    MysqlDataSource mysqlDS = null;

    try {
        fis = new FileInputStream("D:/Assignments/Sequence/db.properties");
    } catch (FileNotFoundException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
    props.load(fis);
    mysqlDS = new MysqlDataSource();
    mysqlDS.setURL(props.getProperty("MYSQL_DB_URL"));
    mysqlDS.setUser(props.getProperty("MYSQL_DB_USERNAME"));
    mysqlDS.setPassword(props.getProperty("MYSQL_DB_PASSWORD"));
    return mysqlDS;
}

//Adding values to Hashset
private static int addNode(){
    set = new HashSet();
    for(int i=1;i<=10000;i++){/*Change 10000 to 30000*/
        set.add(i);
    }
    return 0;
}
private static int keyNode(int i){
    int counter = 1;
    Iterator it = set.iterator();
    while(it.hasNext())
    {
        int value = (int) it.next();
        if(i==counter)
        {
            //System.out.println("key value returned ::"+value);
            return value;
        }
        counter++;
    }
    return 0;
}
private static String pairGenerator(){ 
    ArrayList numbers1 = new ArrayList<Integer>();   
    Random randomGenerator1 = new Random();
    while (numbers1.size() < 1) 
    {
        int random = randomGenerator1 .nextInt(15);
        if (!numbers1.contains(random)) {
            numbers1.add(random);
        }
    }
    Iterator it1 = numbers1.iterator();
    while(it1.hasNext()){
        return(String.valueOf(it1.next()));
    }
    return null;
}

private static List valueNodes(){
    //Generate no randomly.
    ArrayList<Integer> numbers = new ArrayList<Integer>();   
    Random randomGenerator = new Random();
    String size = pairGenerator();
    int size1= Integer.parseInt(size)+1;
    //System.out.println("the size1 is ::"+size1);
    while (numbers.size() < size1) 
    {
        int random = randomGenerator .nextInt(10000);/*Change 10000 to 50000*/
        if (!numbers.contains(random)) {
            numbers.add(random);
        }
    }
    Iterator it = numbers.iterator();
    al3.clear();

    while(it.hasNext()){
        int listvalue = (int) it.next();
        al3.add(listvalue);

        //System.out.println(it.next());
    }
    //System.out.println(al3);
    return al3;
}
public static void main(String[] args) throws Exception {
    Connection con = null;
    PreparedStatement pst = null;
    ResultSet rs = null;
    HashMap<Integer, List<String>> hm = new HashMap<Integer, List<String>>();
    addNode(); 
    //System.out.println("size of set is:"+set.size());
    try {
        con = getMySQLDataSource().getConnection();
        List<Integer> valueList = new ArrayList<Integer>();
        int nodeId;
        for(int i=1;i<=set.size();i++)
        {
            hm.put(keyNode(i), valueNodes());
            Iterator iter = hm.entrySet().iterator();
            while(iter.hasNext())
            {
                Map.Entry entry = (Map.Entry) iter.next();
                System.out.println(entry.getKey()+"<-->"+" "+entry.getValue());
                nodeId = (int) entry.getKey();
                valueList = (List<Integer>) entry.getValue();
                //System.out.println("size of value list : "+valueList.size());
                for(int j = 0;j<valueList.size();j++)
                {
                    pst = con.prepareStatement("insert into nodes_connection values (?,?)");
                    pst.setInt(1, nodeId);
                    if(valueList.get(j)!=0)
                    {
                        pst.setInt(2,valueList.get(j));
                    }
                    else{
                        int updatedValue = valueList.get(j)+10000;/*Change 10000 to 30000*/
                        pst.setInt(2,updatedValue);
                    }
                    pst.executeUpdate();
                    //System.out.println(j+"record updated..");
                }
                iter.remove();
            }   
        }
        System.out.println("Record successfully added");
    } catch (SQLException e) {
        e.printStackTrace();
    }finally{
        try {
            if(rs != null) rs.close();
            if(pst != null) pst.close();
            if(con != null) con.close();
        } catch (SQLException e) {
            e.printStackTrace();
        }
    }


}`

我需要删除arraylist和hashmap。我怎么能优化代码，以便我不会得到任何Java堆内存异常。

Answer 1

由于您要插入这么多行，所以应该使用批量更新，而不是一次插入一行。

PreparedStatement pst = con.prepareStatement("insert into nodes_connection values (?,?)");
 for(int i=1;i<=set.size();i++)
    {
        hm.put(keyNode(i), valueNodes());
        Iterator iter = hm.entrySet().iterator();
        while(iter.hasNext())
        {
            Map.Entry entry = (Map.Entry) iter.next();
            System.out.println(entry.getKey()+"<-->"+" "+entry.getValue());
            nodeId = (int) entry.getKey();
            valueList = (List<Integer>) entry.getValue();
            //System.out.println("size of value list : "+valueList.size());
            for(int j = 0;j<valueList.size();j++)
            {

                pst.setInt(1, nodeId);
                if(valueList.get(j)!=0)
                {
                    pst.setInt(2,valueList.get(j));
                }
                else{
                    int updatedValue = valueList.get(j)+10000;/*Change 10000 to 30000*/
                    pst.setInt(2,updatedValue);
                }
                pst.addBatch()
                //System.out.println(j+"record updated..");
            }
            iter.remove();
        }   
    }


    pst.executeBatch()

您可以前往here了解批量插入的更多信息。

Answer 2

到目前为止，您最简单的事情就是increase the heap size。

如果这只是你要运行一次或正在玩的东西，基本上任何非生产关键或处理大量数据的东西，那么增加堆将给你你想要的东西。

如果确实需要保持内存占用率，则需要流式传输数据而不是将其全部读入内存。查看代码时，似乎每行数据与文件中的其余数据无关，因此以伪代码方式，以下内容将起作用：

For each line in file
    Calculate data to be inserted into database
    Update database

您可以更多地提高效率，而不是为每一行更新数据库，而是批量执行，这会将伪更改为：

For each line in file
    Calculate data to be inserted into database
    Add update to a JDBC batch
    If batch size > :somelimit
         execute batch
Execute final batch

超过1个lac数据的内存不足异常

2 个答案: