使用Netty
我的java代码需要一些帮助。我使用netty 4.1.5 final实现了一个http监听器。监听器将接收的数据转发给Kafka制作人。在测试http监听器和kafka生产者时，一切似乎都很好。但从长远来看，我注意到一些内存泄漏正在发生。
我正在使用具有1500个并行线程的数据生成器（在另一台机器上运行），每个线程每1.5秒生成一个数据包。每个数据包大约1.2 KB。当我刚刚启动监听器而没有轰炸它使用大约50MB的任何数据时（使用TOP命令找到）并且一旦我开始发送数据它开始急剧增加到~2.0GB然后它逐渐减少直到它达到大约900MB并且保持不变。它在大约25-30分钟内增加到2GB，在5分钟内减少到900MB。但奇怪的是，在大约3.5小时不变后，内存使用率再次开始增加，并且达到3.1GB左右。我停止了数据模拟器一分钟然后再次启动它们。这样做后，内存使用量将持续保持在3.1GB，持续4小时。我不知道这是不是特例，因为我没有时间再测试10个小时了。
我检查了GC的过程。我附上了GC最初几分钟的屏幕截图。 令我担心的是，内存最初会爆发，即使我杀死模拟器，内存也不会下降。 我还注意到，完整的GC根本不会发生，至少在我监控GC的前40分钟内。
我在32GB 6核心盒子上运行它，openJDK 7，其他进程共享的centos 6，如kafka，hbase，它们都是活跃的。
Netty版本 - 4.1.5最终版
我只是想知道是否有一些明显的内存泄漏，我错过了。
我已经被困了一段时间了。
提前致谢。
听众代码
public class Http_Listener extends listener implements Runnable {
private ChannelFuture channel;
private final EventLoopGroup masterGroup = new NioEventLoopGroup();
private final EventLoopGroup slaveGroup = new NioEventLoopGroup();
private int port;
Thread t;
Kafka_Producer kafka_producer = new Kafka_Producer("String");
int counter = 1;
public Http_Listener(int port)
{
this.port = port;
}
public void start()
{
t = new Thread(this);
t.start();
}
@Override
public void run() {
//For error logging
PrintWriter pw;
DateFormat dateFormat = new SimpleDateFormat("dd-MM-yyyy");
Date date = new Date();
//will be called when the program shuts down in any manner
Runtime.getRuntime().addShutdownHook(new Thread(){
@Override
public void run() { shutdown(); }
});
try
{
//Logging in new connection
try {
pw=new PrintWriter(new FileOutputStream( new File("//home//hduser//java_codes//nettyDynamicListner//logs//logFile_"+ dateFormat.format(new Date()) + ".txt"),true));
pw.println(date.toString() + " : Connected with port " + port +". - ");
pw.close();
} catch(Exception ex) {
ex.printStackTrace();
System.exit(0);
System.out.println("Error opening the log file");
}
System.out.println("Binded with port " + port);
final ServerBootstrap bootstrap =
new ServerBootstrap()
.group(masterGroup, slaveGroup)
.option(ChannelOption.SO_BACKLOG, 8192)
.channel(NioServerSocketChannel.class)
.childHandler(new ChannelInitializer<SocketChannel>(){
@Override
public void initChannel(final SocketChannel ch) throws Exception
{
ch.pipeline().addLast("codec", new HttpServerCodec());
ch.pipeline().addLast("aggregator", new HttpObjectAggregator(512*1024));
ch.pipeline().addLast("request", new ChannelInboundHandlerAdapter()
{
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception
{
//For stats logging
PrintWriter pw;
DateFormat dateFormat = new SimpleDateFormat("dd-MM-yyyy");
Date date = new Date();
String msgString = "";
String statString;
if (msg instanceof FullHttpRequest)
{
final FullHttpRequest request = (FullHttpRequest) msg;
final String responseMessage = "";
FullHttpResponse response = new DefaultFullHttpResponse(
HttpVersion.HTTP_1_1,
HttpResponseStatus.OK,
copiedBuffer(responseMessage.getBytes())
);
if (request.headers().get("Connection") == "keep-alive"){
response.headers().set("Connection", "keep-alive");
}
response.headers().set("Content-Type", "text/plain");
response.headers().set("Content-Length", responseMessage.length());
ctx.writeAndFlush(response);
response.release();
HttpHeaders headers = request.headers();
for (String name : headers.names()) {
msgString = msgString + name + ":" + headers.get(name) + ";";
}
msgString = msgString + "\n";
msgString = msgString + request.content().toString(CharsetUtil.UTF_8);
//System.out.println(msgString);
//System.out.println(counter++);
kafka_producer.sendMessage("topic_"+Integer.toString(port), msgString);
//Port stats log
statString = date.toString() + "," + ctx.channel().remoteAddress().toString() + "," + msgString.length();
try {
pw=new PrintWriter(new FileOutputStream( new File("//home//hduser//java_codes//nettyDynamicListner//logs//" + port +"_portStats_"+ dateFormat.format(new Date()) + ".txt"),true));
pw.println(statString);
pw.close();
} catch(Exception ex) {
System.out.println("Error opening the log file");
}
request.release();
}
else
{
super.channelRead(ctx, msg);
}
}
@Override
public void channelReadComplete(ChannelHandlerContext ctx) throws Exception
{
ctx.flush();
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception
{
//For error logging
PrintWriter pw;
DateFormat dateFormat = new SimpleDateFormat("dd-MM-yyyy");
Date date = new Date();
//Logging error
try {
pw=new PrintWriter(new FileOutputStream( new File("//home//hduser//java_codes//nettyDynamicListner//logs//logFile_"+ dateFormat.format(new Date()) + ".txt"),true));
pw.println(date.toString() + " : Error in port " + port +". - " + cause.getMessage());
pw.close();
} catch(Exception ex) {
System.out.println("Error opening the log file");
}
ByteBuf responseBuf = copiedBuffer(cause.getMessage().getBytes());
FullHttpResponse response = new DefaultFullHttpResponse(
HttpVersion.HTTP_1_1,
HttpResponseStatus.INTERNAL_SERVER_ERROR,
responseBuf
);
ctx.writeAndFlush(response);
responseBuf.release();
response.release();
}
});
}
})
.option(ChannelOption.TCP_NODELAY, true)
.option(ChannelOption.SO_KEEPALIVE, true);
channel = bootstrap.bind(port).sync();
}
catch (final InterruptedException e) {
//Error logging
try {
pw=new PrintWriter(new FileOutputStream( new File("//home//hduser//java_codes//nettyDynamicListner//logs//logFile_"+ dateFormat.format(new Date()) + ".txt"),true));
pw.println(date.toString() + " : Interrupted exception in port " + port +". - " + e.getMessage());
pw.close();
} catch(Exception ex) {
System.out.println("Error opening the log file");
ex.printStackTrace();
System.exit(0);
}
}
}
@Override
int getPort() {
return port;
}
@Override
boolean isAlive() {
return t.isAlive();
}
public void shutdown()
{
//For error logging
PrintWriter pw;
DateFormat dateFormat = new SimpleDateFormat("dd-MM-yyyy");
Date date = new Date();
slaveGroup.shutdownGracefully();
masterGroup.shutdownGracefully();
try
{
channel.channel().closeFuture().sync();
}
catch (InterruptedException e) { }
//Logging connection disconnect
try {
pw=new PrintWriter(new FileOutputStream( new File("//home//hduser//java_codes//nettyDynamicListner//logs//logFile_"+ dateFormat.format(new Date()) + ".txt"),true));
pw.println(date.toString() + " : Disconnected from port " + port +". - ");
pw.close();
} catch(Exception ex) {
System.out.println("Error opening the log file");
}
}
}
卡夫卡制片人
public class Kafka_Producer {
Properties props;
PrintWriter pw;
DateFormat dateFormat = new SimpleDateFormat("dd-MM-yyyy");
Producer producer;
/* Value of valSerializer should be either 'Byte' or 'String', anything other
* than 'Byte' will be considered as 'String'. This is to avoid the creation
* of a new Producer for each data packet
*/
Kafka_Producer(String valSerializer) {
props = new Properties();
props.put("bootstrap.servers", "servers");
props.put("acks", "1");
props.put("retries", 0);
props.put("batch.size", 800);
props.put("linger.ms", 10);
props.put("buffer.memory", 33554432);
props.put("max.block.ms", 2500);
props.put("timeout.ms", 2000);
props.put("request.timeout.ms", 2000);
props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
//To determine the type of producer to be initialized
if(valSerializer.toLowerCase() == "byte") {
props.put("value.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
producer = new KafkaProducer<String, byte[]>(props);
} else {
props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
producer = new KafkaProducer<String, String>(props);
}
}
//For http messages
public void sendMessage(String topic, String msg) {
try {
/*
* Calling future.get is performance in effective, but since we need
* to make sure every data packet is send successfully else written to a
* file, we take this performance hit.
*/
Future<RecordMetadata> future=producer.send(new ProducerRecord<String, String>(topic, msg));
future.get();
} catch(Exception e) {
System.out.println("exception" + e.getLocalizedMessage());
Date date = new Date();
try {
pw=new PrintWriter(new FileOutputStream( new File("D:\\Work\\nettyDynamicListner\\logs\\" + topic + "_failedMessage_"+ dateFormat.format(new Date()) + ".txt"),true));
pw.println(date.toString() + ":" + topic + ":" + msg);
pw.close();
} catch(Exception ex) {
System.out.println("Error opening the failed message log file");
}
}
}
}
GC LOG
0 个答案:
没有答案
相关问题
最新问题
- 我写了这段代码，但我无法理解我的错误
- 我无法从一个代码实例的列表中删除 None 值，但我可以在另一个实例中。为什么它适用于一个细分市场而不适用于另一个细分市场？
- 是否有可能使 loadstring 不可能等于打印？卢阿
- java中的random.expovariate()
- Appscript 通过会议在 Google 日历中发送电子邮件和创建活动
- 为什么我的 Onclick 箭头功能在 React 中不起作用？
- 在此代码中是否有使用“this”的替代方法？
- 在 SQL Server 和 PostgreSQL 上查询，我如何从第一个表获得第二个表的可视化
- 每千个数字得到
- 更新了城市边界 KML 文件的来源？