Question

我很好奇是否有更快的方法将数据从文件传输到我的postgreSQL数据库。我正在使用java。这些文件是GRIB文件，因此基本上是天气类型文件。我已经创建了一个文本文件，我将获取我正在寻找的文件的名称，并使用它来获取我的数据然后将数据发送到我的数据库。有人能给我一些暗示我应该提高速度吗？

public class JgribDEXcopy{
// Global variables
static GribFile gribfile;
static int numOFrecords;
static int numOFgds;
protected static int eventX;
protected static int eventY;
protected static int xIA = 543;
protected static int yIA = 451;
protected static int xIAend = 703;
protected static int yIAend = 591;
private static Connection connection = null;
private static PreparedStatement state = null;
static String st;
static int k, l, in = 0;
static File file = new File("E:/IFIC/2009.txt");

// Gets the number of records in the GRIB file
// Prints out that number for testing the correct amount of records
public static void read(GribFile n){
    numOFrecords = n.getRecordCount();
    //System.out.println("HELLO: " + numOFrecords + " & " + numOFgds + " Is the magic number");
}


 public static void main(String[] args) throws ClassNotFoundException, SQLException, NoSuchElementException, IOException{


     BufferedReader br = new BufferedReader(new FileReader(file));
     String line = "";
     while((line = br.readLine()) != null){
            int counter = 0;

     // Connecting to the database
     Class.forName("org.postgresql.Driver");
     connection = DriverManager.getConnection("jdbc:postgresql://database/database","username", "password");

     String nline = "E:/IFIC/IIHS_data/ST4." + line + ".01h.grb";

    // Reading of grib files must be inside a try-catch block
    try{
      // Create GribFile instance
      gribfile = new GribFile(nline);
      read(gribfile);

      // Dump verbose inventory for each record
      //gribfile.listRecords(System.out);
      // Dumps the grid/s into an array
      GribRecordGDS[] arr = gribfile.getGrids();
      float arrBDS;

      // Gets the Width and Height of the grid
      // Calculates the number of points in the grid
      eventX = arr[0].getGridNX();
      eventY = arr[0].getGridNY();
      numOFgds = eventX * eventY;
      //System.out.println(numOFgds);
      //System.out.println(eventX);
      //System.out.println(eventY);

      // Creates a writer that will print to a file
      //PrintWriter writer = new PrintWriter("ST4.2014030123.01h.txt", "UTF-8");
      //System.out.println((int) (gribfile.getRecord(1).getTime().getTime().getTime()/1000));

      // Filters for the area Iowa Flood Center deals with (which is basically Iowa area)
      // Takes the database connection and inserts the necessary data
      for(int i = 0; i <= eventX; i++){
          for(int j = 0; j <= eventY; j++){
              if(i >= xIA && i <= xIAend && j <= yIAend && j >= yIA){
                  arrBDS = gribfile.getRecord(1).getValue(i, j);
                  if(arrBDS != 0.000){
                      k = i;
                      l = j;
                      in = 1000 * (k+1) + (l+1);
                      //writer.println(i + "," + j + "," + arrBDS);
                      //System.out.println(in + "," + arrBDS);

//                        st = "INSERT INTO master_st4(unix_timestamp, hrap_xy, rain) values(?, ?, ?)";
//                        //System.out.println((int)(gribfile.getRecord(1).getTime().getTime().getTime()/1000));
//                        
//                        state = connection.prepareStatement(st);
//                        state.setInt(1, (int) (gribfile.getRecord(1).getTime().getTime().getTime()/1000));
//                        state.setInt(2, in);
//                        state.setDouble(3, arrBDS);                    
//                        state.executeUpdate();
                  }
                      // Keeps track of all the X, Y coordinates with zero rainfall
                      if(arrBDS == 0.0){
                      counter++;
                  }
              }
          }
      }
      System.out.println("There is " + counter + " number of zero's");
      //writer.close();
    }
    catch (FileNotFoundException noFileError){
      System.err.println("FileNotFoundException : " + noFileError);
    }
    catch (IOException ioError){
      System.err.println("IOException : " + ioError);
    }
    catch (NoValidGribException noGrib){
      System.err.println("NoValidGribException : " + noGrib);
    }
    catch (NotSupportedException noSupport){
      System.err.println("NotSupportedException : " + noSupport);
    }
  // closes database connection  
  connection.close();
  }
  br.close();
 }
}

Answer 1

使用PgJDBC COPY API批量加载数据。如果做不到这一点，至少要将多值插入分组到事务中。

见：

How to speed up insertion performance in PostgreSQL
http://jdbc.postgresql.org/documentation/publicapi/org/postgresql/PGConnection.html - PGConnection和CopyManager。

使用Java更快地将数据传输到postgreSQL？

1 个答案: