Question

我需要批量导入。文件可以包含1000条记录，每条记录都需要验证。用户希望收到有多少记录无效的通知。最初我用Ruby的Mutex和Redis的发布/订阅做了这个。请注意，我有20个并发线程通过Sidekiq处理每个记录：

  public class MultiThreadChatServerSync {

  // The server socket.
  private static ServerSocket serverSocket = null;
  // The client socket.
  private static Socket clientSocket = null;


  // This chat server can accept up to maxClientsCount clients' connections.
  private static final int maxClientsCount = 50;
  private static final clientThread[] threads = new clientThread[maxClientsCount];

  public static void main(String args[]) {

    // The default port number.
    int portNumber = 2222;
    if (args.length < 1) {
      System.out.println("Usage: java MultiThreadChatServerSync <portNumber>\n"
          + "Now using port number=" + portNumber);
    } else {
      portNumber = Integer.valueOf(args[0]).intValue();
    }

    /*
     * Open a server socket on the portNumber (default 2222). Note that we can
     * not choose a port less than 1023 if we are not privileged users (root).
     */
    try {
      serverSocket = new ServerSocket(portNumber);
      //System.out.println(serverSocket.getPort());
    } catch (IOException e) {
      System.out.println(e);
    }

    /*
     * Create a client socket for each connection and pass it to a new client
     * thread.
     */
    while (true) {
      try {
        clientSocket = serverSocket.accept();
        int i = 0;
        for (i = 0; i < maxClientsCount; i++) {
          if (threads[i] == null) {
            (threads[i] = new clientThread(clientSocket, threads)).start();
            //System.out.println("A new client is created");
            break;
          }
        }
        if (i == maxClientsCount) {
          PrintStream os = new PrintStream(clientSocket.getOutputStream());
          os.println("Server too busy. Try later.");
          os.close();
          clientSocket.close();
        }
      } catch (IOException e) {
        System.out.println(e);
      }
    }
  }
}

class clientThread extends Thread {
MultiThreadChatServerSync ms = new MultiThreadChatServerSync();
  private String clientName = null;
  //private DataInputStream is = null;
   private BufferedReader br = null;
  private PrintStream os = null;
  private Socket clientSocket = null;
  private final clientThread[] threads;
  private int maxClientsCount;

  public clientThread(Socket clientSocket, clientThread[] threads) {
    this.clientSocket = clientSocket;
    this.threads = threads;
    maxClientsCount = threads.length;
    //System.out.println("Inside the Client thread");
  }

  public void run() {
    MultiThreadChatServerSync mss = new MultiThreadChatServerSync();
    int maxClientsCount = this.maxClientsCount;
    clientThread[] threads = this.threads;
    //System.out.println("Inside the run");

    try {
      /*
       * Create input and output streams for this client.
       */
      br = new BufferedReader(new InputStreamReader(clientSocket.getInputStream()));
      os = new PrintStream(clientSocket.getOutputStream());




      while (true) {
        String line = br.readLine();
        System.out.println("message received via Client port: " + clientSocket.getPort());
        System.out.println("Received: " + line);

      }

    } catch (IOException e) {
    }
  }
}

Sidekiq将发布到Record对象：

class Record < ActiveRecord::Base
  class << self
    # invalidated_records is SHARED memory for the Sidekiq worker threads
    attr_accessor :invalidated_records
    attr_accessor :semaphore
  end

  def self.batch_import
  self.semaphore = Mutex.new  
  self.invalid_records = []    
  redis.subscribe_with_timeout(180, 'validation_update') do |on|
    on.message do |channel, message|
      if message.to_s =~ /\d+|import_.+/
        self.semaphore.synchronize {
          self.invalidated_records << message
        }  
      elsif message == 'exit'
        redis.unsubscribe
      end
    end
  end
  end
end

这个问题很奇怪。 Record.invalidated_records中未填充所有无效导入。其中许多都是但不是全部。我以为是因为多个线程试图同时更新对象，它会污染对象。我认为Mutex锁可以解决这个问题。但是在添加Mutex锁之后，并不是所有的invalids都填充在Record.invalidated_records中。

最终，我使用redis原子减量和增量来跟踪无效的导入，这就像一个魅力。但我很好奇Ruby Mutex和多线程试图更新Record.invalidated_records的问题是什么？

Answer 1

我没有使用互斥锁，但我认为发生的事情是线程看到信号量被锁定并跳过保存＆lt;＆lt;信息你需要使用https://apidock.com/ruby/ConditionVariable 等待互斥锁解锁然后保存数据

Ruby中的互斥锁不适用于Redis？

1 个答案: