定期从PostgreSQL获取数据并将其发送到ActiveMQ

时间:2017-03-31 07:22:54

标签: java sql postgresql activemq

基本上,我在PostgreSQL中有一个数据表。 我想获取所有数据,然后发送它成为ActiveMQ的消息(表中1行的1条消息)。

但数据表每5秒自动更新一次。因此,表格中每5秒就有一些新数据。

如何在不发送已发送数据的情况下发送新数据? 而且我想我必须使用Thread.sleep(5000)才能使它无限循环?

这就是我得到的:

package testcode;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import javax.jms.ConnectionFactory;
import javax.jms.Destination;
import javax.jms.JMSException;
import javax.jms.MessageProducer;
import javax.jms.Session;
import javax.jms.TextMessage;
import org.apache.activemq.ActiveMQConnection;
import org.apache.activemq.ActiveMQConnectionFactory;


public class ProducerClear2 {

    public static String vardbserver;
    public static String vardbuser;
    public static String vardbpassword;
    public static String vardbname;

    public static void main (String[] args){
        vardbserver = "TestDBtoMQ";
        vardbuser = "postgresql";
        vardbpassword = "admin";

        ConnectionFactory factory = null;
        javax.jms.Connection connection = null;
        Session session = null;
        Destination destination = null;
        MessageProducer producer = null;
        Connection c = null;
        Statement stmt = null;
        try {
            Class.forName("org.postgresql.Driver");
            c = DriverManager.getConnection("jdbc:postgresql://localhost:5432/"+vardbserver, vardbuser, vardbpassword);
            c.setAutoCommit(false);
            System.out.println("----------------------------");

            stmt = c.createStatement();
            ResultSet rs = stmt.executeQuery("SELECT * FROM MESSAGES;");
            while (rs.next()) {
                String  message = rs.getString("MESSAGE");
                System.out.println("Message = " + message);
                try {
                    factory = new ActiveMQConnectionFactory(ActiveMQConnection.DEFAULT_BROKER_URL);
                    connection = factory.createConnection();
                    connection.start();
                    session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE);
                    destination = session.createQueue("TestQueue");
                    producer = session.createProducer(destination);
                    TextMessage mssg = session.createTextMessage(message);
                    System.out.println("Sent: " + mssg.getText());
                    producer.send(mssg);
                }
                catch (JMSException e) {
                    e.printStackTrace();
                }
            }

            rs.close();
            stmt.close();
            c.close();

        }
        catch (Exception e) {
            System.err.println(e.getClass().getName()+": "+ e.getMessage());
        }finally {
            if (session != null) {
                try {
                    session.close();
                } catch (JMSException ex) {
                    // ignore
                }
            }
            if (connection != null) {
                try {
                    connection.close();
                } catch (JMSException ex) {
                    // ignore
                }
            }
        }
        System.out.println("----------------------------");
        System.out.println("Message sent successfully");
    }
}

答案检查here

2 个答案:

答案 0 :(得分:1)

为了防止重复的消息,有一些可能的方法,例如:

  1. ActiveMQ支持dupplicated message detection,您可以利用此功能。
  2. 使用字段(bool)表示该行是否已发送。
  3. 关于5秒,您可以睡觉或使用调度程序来执行此操作,例如:Spring SchedulerQuartz

答案 1 :(得分:1)

为了从postgres中仅获取新数据,您可以在存储该特定行的事务ID的每一行上使用xmin系统列。

select xmin::varchar::bigint as xrow, * from messages;

然后,您可以跟踪上次提取的交易,以便下次运行查询时可以将该值用作限制

select xmin::varchar::bigint as xrow, * from messages 
    where xmin::varchar::bigint > :last_transaction_id_from_previous_run;

如果您使用的是postgres 9.4或更新版本,您还可以使用逻辑复制来直接从数据库中传输数据。

有关逻辑复制的更多信息:https://www.postgresql.org/docs/current/static/logicaldecoding.html

更新

问题中的示例代码可以像这样重写,为了简洁,我排除了第一个try语句之外的所有内容

// Moved all the setup for the ActiveMQ connection outside the loops
// Otherwise they would create a new connection/sessions for each iteration without being closed
factory = new ActiveMQConnectionFactory(ActiveMQConnection.DEFAULT_BROKER_URL);
connection = factory.createConnection();
connection.start();
session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE);
destination = session.createQueue("TestQueue");
producer = session.createProducer(destination);

Class.forName("org.postgresql.Driver");
System.out.println("----------------------------");

// Made a try-with-resources block in order to auto close everything in case of failure, no need to remember to close them manually.
// Using prepared statements so that the query itself doesn't have to be parsed for each iteration
try (Connection c = DriverManager.getConnection("jdbc:postgresql://localhost:5432/" + vardbserver, vardbuser, vardbpassword);
     PreparedStatement stmt = c.prepareStatement("SELECT * FROM MESSAGES where xmin::varchar::bigint > ? and xmin::varchar::bigint < ? ");
     PreparedStatement max = c.prepareStatement("select max(xmin::varchar::bigint) as txid from messages")
) {
    c.setAutoCommit(false);

    // Keep track of the previous and next transaction id in order to provide windowed results
    Long previousTxId = 0L;
    Long nextTxId = 0L;

    while (true) { // Enter an infinite loop
        stmt.clearParameters(); // Remove any set parameters in the fetch query

        // Fetch the next transaction id by executing the prepared statement in max
        try (ResultSet rs = max.executeQuery()) {
            if (rs.next()) {
                nextTxId = rs.getLong(1);
            }
        }

        // Set the window in the messages query. 
        // Since the query has already been prepared, we only need to set the parameters
        stmt.setLong(1, previousTxId);
        stmt.setLong(2, nextTxId + 1);

        // Execute the statment fetching all messages that were created between previousTxId and nextTxId
        try (ResultSet rs = stmt.executeQuery()) {
            while (rs.next()) {
                String message = rs.getString("MESSAGE");
                System.out.println("Message = " + message);
                TextMessage mssg = session.createTextMessage(message);
                System.out.println("Sent: " + mssg.getText());
                producer.send(mssg);
            }
            // Update previousTxId for the next iteration
            previousTxId = nextTxId; 
        }
        Thread.sleep(5000);
    }
}