Question

我正在构建一个Web服务，它将一个大的“结果集”作为List<List<String>>返回。由于我想避免将XML中的结果集表示为SOAP消息的一部分，我现在将其添加为附件并使用简单的自定义编码。结果集可能非常大，如数百GB。我想避免将内容保留在内存中（因此我使用Iterator<List<String>>而不是List<List<String>>）。此外，我希望能够“管道化”或流式传输数据。特别是，我想要实现的是客户端可以在服务器甚至没有完成写入时开始处理第一个结果。实际上，我并不完全确定这是否可行。事实上，它目前还没有发生。

这是我的服务实现

@MTOM // enable Message Transmission Optimization Mechanism (MTOM) at the server
@WebService ( endpointInterface = "com.SnafucatorWS" )
@StreamingAttachment(parseEagerly = true, memoryThreshold = 4000000L)
public class SnafucatorWSImpl implements SnafucatorWS {
  ...
  @Override
  public GetNodesResponse getNodes() {
    Iterator<List<String>> result = impl.getNodes();    
    return new GetNodesResponse(result);
  }
}

我如何创建客户端连接

  private SnafucatorWS connect() {
    if ( serviceUrl == null ) { 
      throw new UncheckedConnectionException("The service url of the snafucator webservice has not been set"); 
    }
    Service result = null;
    try {
      result = Service.create(
         new URL(serviceUrl + "?wsdl"),
         new QName("http://com/", "SnafucatorWSImplService") );
      } catch ( MalformedURLException e ) {
        LOG.fatal("Could not create web service endpoint.", e);
      }
      SnafucatorWS port = result.getPort(SnafucatorWS.class);
      // enable Message Transmission Optimization Mechanism (MTOM) at the client (for transmission of binary data)
      BindingProvider bindingProvider = (BindingProvider)port;
      SOAPBinding soapBinding = (SOAPBinding)bindingProvider.getBinding();
      soapBinding.setMTOMEnabled(true);    
      return port;
    }

这是我的响应代码以及编码和解码数据的代码：

@XmlAccessorType ( XmlAccessType.NONE )
@XmlRootElement ( name = "getNodesResponse" )
@XmlType ( name = "getNodesResponseType" )
public class GetNodesResponse {
  private static final ExecutorService POOL = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
  private DataHandler results;
  private BlockingQueue<List<String>> resultSet;
  private volatile boolean hasNext;
  private final CountDownLatch streamOpened = new CountDownLatch(1);

  ...

  @XmlElement ( required = true )
  @XmlMimeType ( "*/*" )
  public DataHandler getResults() {
    return results;
  }

 void setResults(DataHandler aDataHandler) {
   results = aDataHandler;
   resultSet = new ArrayBlockingQueue<List<String>>(100);    
   StreamingDataHandler dataHandler = (StreamingDataHandler)results;
   try {
     parse(dataHandler, resultSet);
   } catch ( InterruptedException e ) {
     Thread.currentThread().interrupt();
   }
 }

 /**
  * @return true, if the result set serves more elements
  * @throws InterruptedException if the current thread was interrupted
  */
 public boolean hasNext() throws InterruptedException {
   streamOpened.await();
   return ( hasNext || !resultSet.isEmpty() );
 }

 /**
  * Results can be accessed by successively calling
  * {@link java.util.concurrent.BlockingQueue#take()} as long as {@link #hasNext()} returns true. Please note, however,
  * that an invalid result stream may cause {@link #hasNext()} to return true erroneously, hence the queue might block indefinitely. 
  * 
  * @return the result set as a blocking queue
  */
 public BlockingQueue<List<String>> getResultSet() {
   return resultSet; 
 }

 private DataHandler encode(Iterator<List<String>> aResults) {
   assert ( aResults != null );
   final PipedOutputStream out = new PipedOutputStream();
   DataHandler dh = new DataHandler(new StreamDataSource(out, "*/*"));
   Encoder encoder = new Encoder(out, aResults);
   @SuppressWarnings("unused")
   Future<?> future = POOL.submit(encoder);
   return dh;
 }

 private void parse(StreamingDataHandler dataHandler, final Queue<List<String>> aResultSet) 
     throws InterruptedException {
   Decoder decoder = new Decoder(dataHandler, aResultSet);
   @SuppressWarnings("unused")
   Future<?> future = POOL.submit(decoder);
 }

 ...


private static final class StreamDataSource implements DataSource {
  private final String name = UUID.randomUUID().toString();
  private final InputStream in;
  private final String mimeType;

  private StreamDataSource(PipedOutputStream aOut, String aMimeType) {
    try {
      in = new PipedInputStream(aOut);
    } catch ( IOException e ) {
      throw new RuntimeException("Could not create input stream.", e);
    }
    mimeType = aMimeType;
  }

  @Override public String getName() { return name; }
  @Override public String getContentType() { return mimeType; }

  /**
   * {@inheritDoc}
   * 
   * This implementation violates the specification in that it is destructive. Only the first call will return an
   * appropriate input stream.
   */
  @Override public InputStream getInputStream() { return in; }

  @Override public OutputStream getOutputStream() { throw new UnsupportedOperationException(); }
}

/**
 * Decodes the contents of an input stream as written by the Encoder and writes
 * parsed rows to a {@link java.util.Queue}.
 */
private class Decoder implements Runnable {
  private final StreamingDataHandler dataHandler;
  private final Queue<List<String>> resultSet;

  public Decoder(StreamingDataHandler aDataHandler, Queue<List<String>> aResultSet) {
    dataHandler = aDataHandler;
    resultSet = aResultSet;
  }

  @Override
  public void run() {
    try { 
      InputStream in = dataHandler.getInputStream();
      byte[] lenBytes = new byte[4];
      List<String> row;
      int rowLen;
      // read the first row's length
      in.read(lenBytes);
      rowLen = ByteBuffer.wrap(lenBytes).getInt();
      if ( rowLen == 0 ) { 
        hasNext = false;
        streamOpened.countDown();
      } else {
        hasNext = true;
        streamOpened.countDown(); // now the client can start processing / waiting for rows
        do {
          rowLen = ByteBuffer.wrap(lenBytes).getInt();
          if ( rowLen == 0 ) { break; }
          row = new ArrayList<String>(rowLen);       // read row length
          for ( int col = 0; col < rowLen; col++ ) { // for each column
            in.read(lenBytes);                       // read the value length
            int valLen = ByteBuffer.wrap(lenBytes).getInt();
            byte[] valBytes = new byte[valLen];      // allocate a buffer of appropriate size
            in.read(valBytes);
            row.add(new String(valBytes));
          }
          resultSet.add(row);
        } while ( in.read(lenBytes) > 0 );
        hasNext = false;
      }
    } catch ( IOException e ) {
      throw new RuntimeException(e); 
    } finally {
      hasNext = false;
      try {
        dataHandler.close();
      } catch ( IOException e ) {
        throw new RuntimeException("Could not close data handler.", e); 
      }
    }
  }
}

/**
 * Encodes the given result set and writes the result to an output stream. 
 */
private class Encoder implements Runnable {
  private final OutputStream out;
  private final Iterator<List<String>> iterator;

  public Encoder(OutputStream aOut, Iterator<List<String>> aResults) {
    out = aOut;
    iterator = aResults;
  }

  @Override
  public void run() {
    try { 
      while ( iterator.hasNext() ) {
        List<String> row = iterator.next();
        out.write(ByteBuffer.allocate(4).putInt(row.size()).array());
        for ( String s : row ) {
          byte[] bytes = s.getBytes();
          out.write(ByteBuffer.allocate(4).putInt(s.length()).array()); // write size of column in bytes
          out.write(bytes);                                             // write column value            
        }
      }
    } catch ( IOException e ) {
      throw new RuntimeException("Could not write data.", e); 
    } finally {
      try {
        out.close();
      } catch ( IOException e ) {
        throw new RuntimeException("Could not close data handler.", e); 
      }
    }
  }
}

Web服务返回一个包含完全预期结果集的响应。但是，当我调试时，我发现客户端只能在服务关闭输出流后才开始解码。我的目标是解码器开始工作（在客户端），而编码器任务仍然写入服务器端的输出流。也许这不能修复，但我认为这将是一个很好的性能改进。

提前感谢任何有关如何解决此问题的建议。

Answer 1

问题已解决。除了我的代码中的一些锁定错误（与原始帖子无关），我有错误的注释和客户端配置。这就是一切正常的方式：

服务界面

@WebService
public interface SnafucatorWS {
  @WebMethod ( operationName = "getNodes" )
  public GetNodesResponse getNodes();
}

服务实施

@MTOM // enable Message Transmission Optimization Mechanism (MTOM) at the server
@WebService ( endpointInterface = "com.snafu.SnafucatorWS" )
@StreamingAttachment(parseEagerly = false, dir = "/tmp", memoryThreshold = 4000000L)
public class SnafucatorWSImpl implements SnafucatorWS {
  @Override public GetNodesResponse getNodes() { ... }
}

客户端中的连接配置

public final class SnafucatorWSClient implements SnafucatorWS {
  private SnafucatorWS connect() {
    Service result = null;
    try {
      result = Service.create(
        new URL(serviceUrl + "?wsdl"),
        new QName("http://snafu.com/", "SnafucatorWSImplService") );
    } catch ( MalformedURLException e ) {
      LOG.fatal("Could not create web service endpoint.", e);
    }
    // load off attachment to the file system when exceeding 4MB in size
    StreamingAttachmentFeature stf = new StreamingAttachmentFeature("/tmp", false, 4000000L);
    SnafucatorWS port = result.getPort(SnafucatorWS.class, new MTOMFeature(), stf);
    // enable Message Transmission Optimization Mechanism (MTOM) at the client (for transmission of binary data)
    BindingProvider bindingProvider = (BindingProvider)port;
    SOAPBinding soapBinding = (SOAPBinding)bindingProvider.getBinding();
    soapBinding.setMTOMEnabled(true);
    return port;
  }
}

结果类型

@StreamingAttachment(parseEagerly=false, memoryThreshold=40000L)
@XmlAccessorType ( XmlAccessType.NONE )
@XmlRootElement ( name = "getNodesResponse" )
@XmlType ( name = "getNodesResponseType" )
public class GetNodesResponse {
  @XmlElement ( required = true )
  @XmlMimeType("application/octet-stream")
  DataHandler getResults() {
    return results;
  }

  void setResults(DataHandler aDataHandler) { ... /* do the parallel parsing here */ }
}

现在，客户端在服务器关闭流之前解析结果。

流式传输大型SOAP附件

1 个答案: