commit 62902dd37cffb84d39f78fbc8daaf13458cccfdd Author: y Date: Sat Apr 23 15:01:52 2016 +0800 mom diff --git a/.classpath b/.classpath new file mode 100644 index 0000000..e5f91ee --- /dev/null +++ b/.classpath @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.project b/.project new file mode 100644 index 0000000..61c5d0e --- /dev/null +++ b/.project @@ -0,0 +1,29 @@ + + + middleware-mom + + + + + + org.eclipse.wst.common.project.facet.core.builder + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + org.eclipse.wst.common.project.facet.core.nature + + diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..4c28b1a --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 +encoding//src/test/java=UTF-8 +encoding/=UTF-8 diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..bbcbc93 --- /dev/null +++ b/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 +org.eclipse.jdt.core.compiler.compliance=1.7 +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.source=1.7 diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..14b697b --- /dev/null +++ b/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/broker.sh b/broker.sh new file mode 100644 index 0000000..c134c86 --- /dev/null +++ b/broker.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +java -cp target/lib/middleware-mom-1.0.jar com.alibaba.middleware.race.mom.Broker diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..1cc9eb0 --- /dev/null +++ b/build.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +mvn clean package diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..71ed924 --- /dev/null +++ b/pom.xml @@ -0,0 +1,107 @@ + + 4.0.0 + + com.alibaba.race + middleware-mom + jar + + middleware-mom + http://maven.apache.org + + + UTF-8 + + + + ${basedir}/target/lib + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 1.7 + 1.7 + UTF-8 + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.4 + + false + + src/main/java/assembly.xml + + + jar-with-dependencies + + + + + make-assembly + + package + + + single + + + + + + + + + + com.alibaba + fastjson + 1.2.5 + + + net.minidev + json-smart + 2.0 + + + de.ruedigermoeller + fst + 2.34 + + + com.esotericsoftware + kryo + 3.0.2 + + + + io.netty + netty-all + 4.0.23.Final + + + + org.slf4j + slf4j-log4j12 + 1.7.2 + + + + + log4j + log4j + 1.2.12 + + + + + junit + junit + 4.7 + test + + + 1.0 + diff --git a/src/main/java/assembly.xml b/src/main/java/assembly.xml new file mode 100644 index 0000000..173e76c --- /dev/null +++ b/src/main/java/assembly.xml @@ -0,0 +1,30 @@ + + bin + middleware-mom + true + + dir + + + + ${project.basedir} + / + + README* + LICENSE* + NOTICE* + + + + ${project.basedir}/lib + lib + + + + + /lib + true + runtime + + + \ No newline at end of file diff --git a/src/main/java/com/alibaba/middleware/race/mom/Broker.java b/src/main/java/com/alibaba/middleware/race/mom/Broker.java new file mode 100644 index 0000000..a3a8a37 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/Broker.java @@ -0,0 +1,276 @@ +package com.alibaba.middleware.race.mom; + +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.nio.NioServerSocketChannel; + +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.middleware.race.mom.broker.function.MessageManager; +import com.alibaba.middleware.race.mom.broker.function.MessageSend; +import com.alibaba.middleware.race.mom.broker.group.ConsumerGroup; +import com.alibaba.middleware.race.mom.serializer.RpcDecoder; +import com.alibaba.middleware.race.mom.serializer.RpcEncoder; +import com.alibaba.middleware.race.mom.util.InfoBodyConsumer; +import com.alibaba.middleware.race.mom.util.TopicAndFilter; + + +public class Broker { + private static Logger logger = LoggerFactory.getLogger(Broker.class); + /** + * 启动broker之前的netty服务器处理 + * @param port + * @throws Exception + */ + public void bind(int port) throws Exception { + //启动消费进度定时任务 +// storeSubscribe(true, 5000); +// scanSendInfotMap(); + EventLoopGroup bossGroup = new NioEventLoopGroup(); + EventLoopGroup workerGroup = new NioEventLoopGroup(Runtime.getRuntime().availableProcessors()*3); + try { + ServerBootstrap serverBootstrap = new ServerBootstrap(); + serverBootstrap.group(bossGroup, workerGroup).channel(NioServerSocketChannel.class) + .option(ChannelOption.SO_REUSEADDR, true) + .childOption(ChannelOption.TCP_NODELAY, true) + .option(ChannelOption.SO_BACKLOG, 1024*1024) + .childOption(ChannelOption.SO_KEEPALIVE, true) + // +// .option(ChannelOption.SO_BACKLOG, 1024) +// // +// .option(ChannelOption.SO_REUSEADDR, true) +// // +// .option(ChannelOption.SO_KEEPALIVE, false) +// // +// .childOption(ChannelOption.TCP_NODELAY, true) +// .option(ChannelOption.SO_SNDBUF, 65535) +// // +// .option(ChannelOption.SO_RCVBUF, 65535) + .childHandler(new ChannelInitializer() { + @Override + protected void initChannel(Channel ch) throws Exception { + ch.pipeline().addLast(new RpcDecoder()).addLast(new RpcEncoder()) + .addLast(new SimpleChannelInboundHandler() { + @Override + protected void channelRead0(ChannelHandlerContext ctx, Object info) throws Exception { + if (InfoBodyConsumer.class.isInstance(info)) { + // 接受消费者订阅处理 + processConsumer((InfoBodyConsumer) info, ctx); + } else if (ConsumeResult.class.isInstance(info)) { + // 收到消费者ConsumeResult + logger.debug(" 收到消费者ConsumeResult"); + ConsumerGroup.confirmConsumer((ConsumeResult)info, ctx); +// confirmConsumer((ConsumeResult) info, ctx); + }else if(MessageSend.class.isInstance(info)){ + //System.out.println("收到消息"); + MessageManager.recieveMsg((MessageSend) info,ctx); +// MessageManager.recieveMsg((LinkedBlockingQueue)info,ctx); + } + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + logger.error("broker 异常："); +// logger.error(cause.getMessage()); + cause.printStackTrace(); + ctx.close(); + } + }); + } + });// + + ChannelFuture future = serverBootstrap.bind(port).sync(); + logger.debug("mom服务启动成功...... 绑定端口" + port); + + // 等待服务端监听端口关闭 + future.channel().closeFuture().sync(); + } catch (InterruptedException e) { + logger.error("smom服务抛出异常 " + e.getMessage()); + } finally { + // 优雅退出 释放线程池资源 + bossGroup.shutdownGracefully(); + workerGroup.shutdownGracefully(); + logger.debug("mom服务优雅的释放了线程资源..."); + } + + } + /** + * 收到消费者的订阅关系，并创建队列 + * @param consumerRequestInfo + * @param ctx + */ + public void processConsumer(InfoBodyConsumer consumerRequestInfo, final ChannelHandlerContext ctx) { + + String topic = consumerRequestInfo.getTopic(); + String groupid = consumerRequestInfo.getGroupId(); + Map filter = consumerRequestInfo.getFilterMap(); + TopicAndFilter topicAndFilter = new TopicAndFilter(topic, filter); + Channel consumer = ctx.channel(); + logger.error("消费者:" + consumer.hashCode()+"发起订阅"+consumerRequestInfo); + ConsumerGroup.addConsumer(consumer,groupid,topicAndFilter); +// + } + +// public void scanSendInfotMap(){ +// this.timoutScannerTimer.schedule(new TimerTask(){ +// @Override +// public void run() { +// for(Map.Entry entry:msgManager.getSendInfoMap().entrySet()){ +// SendInfo sendInfo=entry.getValue(); +// for(Map.Entry groupstatus:sendInfo.getGroup2status().entrySet()){ +// if(groupstatus.getValue()==SendStatus.TIMEOUT||sendInfo.isTimeout(System.currentTimeMillis())){ +// String msgId=sendInfo.getMsgId(); +// final String topic=sendInfo.getTopic(); +// final TopicAndFilter topicAndfilter=sendInfo.getTopicAndFilter(); +// final ConsumerGroup group=groupstatus.getKey(); +// BlockingQueue reSendMessageQueue=topicAndgroup2reSendMsgQueue.get(topic+ "@" +group); +// if(reSendMessageQueue==null){ +// reSendMessageQueue=new LinkedBlockingQueue(); +// reSendMessageQueue.add(msgId); +// topicAndgroup2reSendMsgQueue.put(topic+ "@" +group,reSendMessageQueue); +// }else { +// reSendMessageQueue.add(msgId); +// } +// // messageCache.remove(msgId);//消息进入重试队列，这里应该移除了，sendResultMap还可以重试结果使用 +// reSendPool.execute(new Runnable(){ +// @Override +// public void run() { +// reSendToConsumer(topicAndgroup2reSendMsgQueue.get(topic+ "@" +group),group); +// } +// }); +// // if(msgid2offset.containsKey(msgId)){ +// // topicAndgroup2subScribe.get(topic+"@"+group).get(sendInfo.getQueueId()).setCurrentoffset(msgid2offset.get(msgId)); +// // msgid2offset.remove(msgId); +// // } +// } +// } +// } +// +// } +// },timeoutLimit+100,1000); +// } + /** + * 恢复订阅关系 + */ +// public void recover(){ +// logger.debug("恢复订阅关系队列"); +// long startTime = System.currentTimeMillis(); +// ArrayListoffsets =substore.read(); +// System.out.println("offsets:"+offsets); +// String topic; +// String group; +// String queueId; +// String topicAndFilter; +// for(Offset offset:offsets){ +// topic=offset.getTopicId(); +// topicAndFilter=offset.getTopicAndFilter(); +// group=offset.getGroupId(); +// queueId=offset.getQueueId(); +// //恢复topicAndgroup2subScribe +// LinkedList offsetList; +// if(!topicAndgroup2subScribe.containsKey(topic+"@"+group)) +// { +// offsetList=new LinkedList(); +// topicAndgroup2subScribe.put(topic+"@"+group, offsetList); +// }else +// { +// offsetList=topicAndgroup2subScribe.get(topic+"@"+group); +// } +// offsetList.add(Integer.parseInt(queueId),offset); +// //恢复topicAndgroup2sendMsgQueue +// LinkedList> sendQueuesList; +// if(!topicAndgroup2sendMsgQueue.containsKey(topic + "@" + group)) +// { +// sendQueuesList=new LinkedList>(); +// topicAndgroup2sendMsgQueue.put(topic + "@" + group, sendQueuesList); +// }else +// { +// sendQueuesList=topicAndgroup2sendMsgQueue.get(topic + "@" + group); +// } +// BlockingQueuesendQueue=new LinkedBlockingQueue<>(); +// sendQueuesList.add(Integer.parseInt(queueId),sendQueue); +// //消息入队(这里只磁盘拉) +// int currentOffset=offset.getCurrentoffset(); +// Message msg; +// int i=0;//currentOffset偏移量 +// List blist = mstore.readByteNormal(topicAndFilter,"t","t",queueId+"",offset.getCurrentoffset()+1, offset.getCacheOffset()-1); +// System.out.println(blist.size()); +// for(byte[] b:blist){ +// try{ +// msg=(Message)fst.asObject(b); +// }catch(Exception e){ +// continue; +// } +// System.out.println(msg); +// sendQueue.add(msg.getOffset()); +// msgManager.getMessageCacheMap().put(msg.getOffset(), msg); +// msgid2offset.put(msg.getMsgId(), currentOffset+i); +// i++; +// } +// +// } +// +// long endTime = System.currentTimeMillis(); +// System.out.println("恢复cost：" + (endTime - startTime)); +// } + /** + * 仅更新到磁盘 Subscribe之前的 offset,这部分属于 离散重试(至少中间有一条成功) ,须重试offset表存储到磁盘,broker故障重启只能离散随机恢复消息 + * 如果存在Subscribe之后的 offset,必然发生连续>=1(甚至大量)超时,连续超时将随Subscribe存储到磁盘,broker故障重启将发生连续一片(>=1条)消息恢复 + * @param start + * @param during + */ + public void storeResendOffset(boolean start,long during/*ms*/){ + + } + /** + * 定时任务-存储订阅关系 + * @param start + * @param during + */ +// public void storeSubscribe(boolean start,long during/*ms*/) +// { +// if(start==false) +// return; +// logger.debug("异步-订阅关系/消费进度定时持久化任务"); +// +// shceduledPool.scheduleWithFixedDelay( +// new Runnable() { +// @Override +// public void run() { +// ArrayList sublist=new ArrayList<>(); +// logger.error("遍历订阅关系:"+topicAndgroup2subScribe.size()); +// for (Map.Entry> entry : topicAndgroup2subScribe.entrySet()) +// { +// //System.out.println("key= " + entry.getKey() + " and value= " + entry.getValue()); +// sublist.addAll(entry.getValue()); +// } +// substore.write(sublist); +// logger.error("订阅关系刷盘"); +// } +// }, +// 5000, +// during, +// TimeUnit.MILLISECONDS); +// } + public static void main(String[] args) throws Exception { + + int port = 9999; + Broker broker=new Broker(); +// Recover recover=new Recover(); +// recover.recover(); +// SubsStore.subscribeStoreStart(); + broker.bind(port); + + } + +} \ No newline at end of file diff --git a/src/main/java/com/alibaba/middleware/race/mom/ConsumeResult.java b/src/main/java/com/alibaba/middleware/race/mom/ConsumeResult.java new file mode 100644 index 0000000..9616823 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/ConsumeResult.java @@ -0,0 +1,49 @@ +package com.alibaba.middleware.race.mom; + +import java.io.Serializable; + +import com.alibaba.middleware.race.mom.util.TopicAndFilter; + +public class ConsumeResult implements Serializable { + private static final long serialVersionUID = -4489150726886715441L; + private ConsumeStatus status=ConsumeStatus.FAIL; + private String info; + private String msgId; + //来自哪个订阅集群 + private String groupId; + //来自哪 些 订阅集群组 + private TopicAndFilter topicAndFilter; + + + public String getGroupId() { + return groupId; + } + public void setGroupId(String groupId) { + this.groupId = groupId; + } + public TopicAndFilter getTopicAndFilter() { + return topicAndFilter; + } + public void setTopicAndFilter(TopicAndFilter topicAndFilter) { + this.topicAndFilter = topicAndFilter; + } + public void setStatus(ConsumeStatus status) { + this.status = status; + } + public ConsumeStatus getStatus() { + return status; + } + public void setInfo(String info) { + this.info = info; + } + public String getInfo() { + return info; + } + public String getMsgId() { + return msgId; + } + public void setMsgId(String msgId) { + this.msgId = msgId; + } + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/ConsumeStatus.java b/src/main/java/com/alibaba/middleware/race/mom/ConsumeStatus.java new file mode 100644 index 0000000..0cbb47e --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/ConsumeStatus.java @@ -0,0 +1,6 @@ +package com.alibaba.middleware.race.mom; + +public enum ConsumeStatus { + SUCCESS, + FAIL +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/Consumer.java b/src/main/java/com/alibaba/middleware/race/mom/Consumer.java new file mode 100644 index 0000000..2ab3628 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/Consumer.java @@ -0,0 +1,56 @@ +package com.alibaba.middleware.race.mom; + +import java.util.Map; + +import com.alibaba.middleware.race.mom.util.InfoBodyConsumer; + +public interface Consumer { + /** + * 启动消费者，初始化底层资源。要在属性设置和订阅操作发起之后执行 + */ + void start(); + + /** + * 发起订阅操作 + * + * @param topic + * 只接受该topic的消息 + * @param filter + * 属性过滤条件，例如 area=hz，表示只接受area属性为hz的消息。消息的过滤要在服务端进行 + * @param listener + */ + void subscribe(String topic, String filter, MessageListener listener); + + /** + * 设置消费者组id，broker通过这个id来识别消费者机器 + * + * @param groupId + */ + void setGroupId(String groupId); + + /** + * 停止消费者，broker不再投递消息给此消费者机器。 + */ + void stop(); + + void prepare(); + + String getGroupId(); + + Map getFilterMap(); + + void setFilterMap(Map filterMap); + + Message getMsg(); + + void setMsg(Message msg); + + InfoBodyConsumer getConsumerRequestInfo(); + + void setConsumerRequestInfo(InfoBodyConsumer consumerRequestInfo); + + String getTopic(); + + void setTopic(String topic); + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/DefaultConsumer.java b/src/main/java/com/alibaba/middleware/race/mom/DefaultConsumer.java new file mode 100644 index 0000000..79fc1ad --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/DefaultConsumer.java @@ -0,0 +1,270 @@ +package com.alibaba.middleware.race.mom; + +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.nustaq.serialization.FSTConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.middleware.race.mom.serializer.RpcDecoder; +import com.alibaba.middleware.race.mom.serializer.RpcEncoder; +import com.alibaba.middleware.race.mom.util.InfoBodyConsumer; +import com.alibaba.middleware.race.mom.util.TopicAndFilter; + +public class DefaultConsumer implements Consumer, Serializable { + + private static final long serialVersionUID = -4454001515077708258L; + private static Logger logger = LoggerFactory.getLogger(DefaultConsumer.class); + private String groupId; + private Map filterMap; + transient private ChannelFuture future; + transient private NioEventLoopGroup group; + transient private String SIP; + private Message msg; + private InfoBodyConsumer consumerRequestInfo; + private MessageListener listener; + private String topic; + FSTConfiguration fst=FSTConfiguration.getDefaultConfiguration(); + Bootstrap bootstrap; + boolean reconnect=false; + @Override + public void start() { + prepare(); + } + + + @Override + public void prepare() { + SIP=System.getProperty("SIP"); + if(SIP==null) + SIP="127.0.0.1"; + System.out.println("consumer connect:"+System.getProperty("SIP")); + group = new NioEventLoopGroup(); + try { + bootstrap = new Bootstrap(); + bootstrap.group(group).channel(NioSocketChannel.class) + .option(ChannelOption.SO_REUSEADDR, true).option(ChannelOption.SO_KEEPALIVE, true) +// .option(ChannelOption.TCP_NODELAY, true) +// // +// .option(ChannelOption.SO_KEEPALIVE, false) +// // +// .option(ChannelOption.SO_SNDBUF, 65535) +// // +// .option(ChannelOption.SO_RCVBUF,65535) + .handler(new ChannelInitializer() { + @Override + public void initChannel(SocketChannel channel) throws Exception { + channel.pipeline().addLast(new RpcEncoder()) + .addLast(new RpcDecoder()) + .addLast(new SimpleChannelInboundHandler() { + + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + connect(); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable e) + throws Exception { + logger.error("消费者异常关闭："); + e.printStackTrace(); + ctx.close(); + + } + @Override + protected void channelRead0(final ChannelHandlerContext ctx, Object info) + throws Exception { + logger.debug("client receive msg"); + Message msg =(Message)info; + //返回ACK + final ConsumeResult consumeResult=listener.onMessage(msg); + //设置谁的 ack + consumeResult.setMsgId(msg.getMsgId()); + /* + *设置 consumeResult来源 + */ + consumeResult.setGroupId(groupId); + consumeResult.setTopicAndFilter(new TopicAndFilter(topic,filterMap)); +// if(Math.random()>0.95){ +// new Thread(new Runnable(){ +// +// @Override +// public void run() { +// // TODO Auto-generated method stub +// try { +// Thread.sleep(10000); +// +// ctx.writeAndFlush(consumeResult); +// } catch (InterruptedException e) { +// // TODO Auto-generated catch block +// e.printStackTrace(); +// } +// } +// +// }).start();; +// }else + ctx.writeAndFlush(consumeResult); + + } + }); + } + }); + connect(); + } catch (InterruptedException e) { + logger.error("消费者抛出异常 " + e.getMessage()); + } + } + + public void connect() throws InterruptedException + { + try{ + future = bootstrap.connect(SIP, 9999).sync(); + future.channel().writeAndFlush(consumerRequestInfo); + logger.error("连接成功"); + }catch(Exception e) + { +// Thread.sleep(1000); + logger.error("重新连接"); + connect(); + } + } + @Override + public void subscribe(String topic, String filter, MessageListener listener) { + this.topic=topic; + this.listener = listener; + if (null != filter && (!"".equals(filter))) { + filterMap = new HashMap(); + String[] temp = filter.split("=",2); + filterMap.put(temp[0],temp[1]); + this.setFilterMap(filterMap); + /*for(int i=0;i getFilterMap() { + return filterMap; + } + + + @Override + public void setFilterMap(Map filterMap) { + this.filterMap = filterMap; + } + + @Override + public Message getMsg() { + return msg; + } + + + @Override + public void setMsg(Message msg) { + this.msg = msg; + } + + + @Override + public InfoBodyConsumer getConsumerRequestInfo() { + return consumerRequestInfo; + } + + + @Override + public void setConsumerRequestInfo(InfoBodyConsumer consumerRequestInfo) { + this.consumerRequestInfo = consumerRequestInfo; + } + + + @Override + public String getTopic() { + return topic; + } + + + @Override + public void setTopic(String topic) { + this.topic = topic; + } + + + @Override + public void stop() { + future.channel().closeFuture(); + future.channel().close(); + group.shutdownGracefully(); + logger.debug("消费者的释放了线程资源..."); + } + + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((filterMap == null) ? 0 : filterMap.hashCode()); + result = prime * result + ((groupId == null) ? 0 : groupId.hashCode()); + result = prime * result + ((topic == null) ? 0 : topic.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + DefaultConsumer other = (DefaultConsumer) obj; + if (filterMap == null) { + if (other.filterMap != null) + return false; + } else if (!filterMap.equals(other.filterMap)) + return false; + if (groupId == null) { + if (other.groupId != null) + return false; + } else if (!groupId.equals(other.groupId)) + return false; + if (topic == null) { + if (other.topic != null) + return false; + } else if (!topic.equals(other.topic)) + return false; + return true; + } + +} \ No newline at end of file diff --git a/src/main/java/com/alibaba/middleware/race/mom/DefaultProducer.java b/src/main/java/com/alibaba/middleware/race/mom/DefaultProducer.java new file mode 100644 index 0000000..735fd61 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/DefaultProducer.java @@ -0,0 +1,328 @@ +package com.alibaba.middleware.race.mom; + +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.handler.codec.LineBasedFrameDecoder; +import io.netty.handler.codec.string.StringDecoder; + +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.middleware.race.mom.broker.function.MessageSend; +import com.alibaba.middleware.race.mom.serializer.RpcEncoder; + + +public class DefaultProducer implements Producer { + transient private static Logger logger = LoggerFactory.getLogger(DefaultProducer.class); + private String topic; + private String groupId; + private static String producerId="pro_"+String.valueOf(Math.random()*100); + private ChannelFuture future; + private static Random random=new Random(); + private NioEventLoopGroup group; + private String SIP; + private SendCallback callback; + private ExecutorService sendPool = Executors.newFixedThreadPool(1); + private BlockingQueue sendQueue=new LinkedBlockingQueue(); + private Map> resultMap=new ConcurrentHashMap>(); + private Bootstrap bootstrap; + boolean isGroupMsg=false; + private AtomicLong msgIdProduce=new AtomicLong(); + private CountDownLatch sendWait=new CountDownLatch(1); + private Object lock=new Object(); + private Map asyncResults=new HashMap<>(); + + + public DefaultProducer() { + + } + + + + @Override + public void asyncSendMessage(Message message, SendCallback callback) { + Message msg = buildMessage(message, callback,1); + //加入同步等待队列 + logger.debug("生产者准备发起请求和消息"); +// resultMap.put(message.getMsgId(), new LinkedBlockingQueue(1)); + asyncResults.put(msg.getMsgId(),callback); + MessageSend msgSend=new MessageSend(); + msgSend.getSendIds().add(msg.getMsgId()); + future.channel().writeAndFlush(msgSend); + } + + private AtomicInteger msgCount=new AtomicInteger(); + @Override + public SendResult sendMessage(Message message) { + int msgCountL=msgCount.incrementAndGet(); + Message msg = buildMessage(message, null,"ackGroup", msgCountL); + resultMap.put(message.getMsgId(), new LinkedBlockingQueue(1)); + sendQueue.add(msg); + if(msgCountL>199){ + sendWait.countDown(); + } + SendResult sr=null; + try { + sr = resultMap.get(message.getMsgId()).poll(5000, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + e.printStackTrace(); + } + if(sr==null) + { + sr=new SendResult(); + sr.setStatus(SendStatus.FAIL); + } +// synchronized(lock){ +// try { +// lock.wait(); +// } catch (InterruptedException e) { +// // TODO Auto-generated catch block +// e.printStackTrace(); +// } +// } + return sr; + } + + /** + * 对Message 进行必要的信息设置,测试设置了groupId和topic producer.setGroupId(PID+code);producer.setTopic(topic); + * @param msg + * @param callback + * @return + */ + private Message buildMessage (Message msg, SendCallback callback,int count) { + if (callback != null) { + this.callback = callback; + logger.debug("生产者异步发送消息"); + } + msg.setMsgId(String.valueOf(Math.random())+" "+String.valueOf(count)); + msg.setTopic(topic); + msg.setBornTime(System.currentTimeMillis()); + return msg; + } + private Message buildMessage (Message msg, SendCallback callback,String qianzhui,int count) { + if (callback != null) { + this.callback = callback; + logger.debug("生产者异步发送消息"); + } + //msg.setMsgId(qianzhui+" "+String.valueOf(count)); + msg.setMsgId(String.valueOf((int)(Math.random()*999%999))+" "+String.valueOf(count)); + msg.setTopic(topic); + msg.setBornTime(System.currentTimeMillis()); + return msg; + } + @Override + public void start() { + SIP=System.getProperty("SIP"); + if(SIP==null) + SIP="127.0.0.1"; + System.out.println("connect:"+System.getProperty("SIP")); + group = new NioEventLoopGroup(); + bootstrap = new Bootstrap(); + bootstrap.group(group).channel(NioSocketChannel.class) + .option(ChannelOption.SO_REUSEADDR, true).option(ChannelOption.SO_KEEPALIVE, true) + .option(ChannelOption.TCP_NODELAY, true) +// // + .option(ChannelOption.SO_KEEPALIVE,true) +// // +// .option(ChannelOption.SO_SNDBUF, 65535) +////// // +// .option(ChannelOption.SO_RCVBUF,65535) + .handler(new ChannelInitializer() { + @Override + public void initChannel(SocketChannel channel) throws Exception { + channel.pipeline() + .addLast(new RpcEncoder()) + .addLast(new LineBasedFrameDecoder(1024*256)) + .addLast(new StringDecoder()) + .addLast(new SimpleChannelInboundHandler() { + + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + logger.error("producer失去broker链接"); + connect(); + } + @Override + protected void channelRead0(ChannelHandlerContext arg0, Object info) throws Exception { + + logger.debug("recv ack： "+(String)info); + + if(info!=null/*&&info.toString().startsWith("ackGroup"*/){ + String id[]=((String)info).split("@"); + //System.out.println("收到:"+info.toString()); + //System.out.println(id.length+":len:"+id[0]+" | "+id[1]); + if(id[0].startsWith("fail")) + { + //System.out.println("无人订阅"); + SendResult sr=new SendResult(); + sr.setMsgId(id[1]); + resultMap.get(id[1]).put(sr); + + }else + { + for(int i=1;i199){ + break; + }else if (msg==null&&msgSend.getBodys().size()==1){ + try {//很有可能只有一条消息，比如函数测试，再确认一下 + msg = sendQueue.poll(10,TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + e.printStackTrace(); + } + if(msg==null){ + logger.error("10ms no message produced after last first message"); + break; + } + }else if(msg==null){ + continue; + } + if(msgSend.getTopic()==null||msgSend.getProperties()==null){ + msgSend.setProperties(msg.getProperties()); + msgSend.setTopic(msg.getTopic()); + } + msgSend.getBodys().add(msg.getBody()); + msgSend.getSendIds().add(msg.getMsgId()); + } + if(msgSend.getBodys().size()>0){ + + //msgSend.setSendId("ackGroup "+String.valueOf(msgSend.getBodys().size())); +// logger.error((System.currentTimeMillis()-start)+"后生产者一次打包发送"+msgSend.getBodys().size()); + future.channel().writeAndFlush(msgSend); + msgCount.set(0); + sendWait=new CountDownLatch(1); + } + } + } + + }); + + } + @Override + public void stop() { + future.channel().closeFuture(); + future.channel().close(); + group.shutdownGracefully(); + sendPool.shutdown(); + } + + + @Override + public void setGroupId(String groupId) { + // TODO Auto-generated method stub + this.groupId = groupId; + } + + @Override + public void setSIP(String sip) { + this.SIP = sip; + } + + @Override + public void setTopic(String topic) { + this.topic = topic; + } + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/DefaultProducer1.java b/src/main/java/com/alibaba/middleware/race/mom/DefaultProducer1.java new file mode 100644 index 0000000..89e7a48 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/DefaultProducer1.java @@ -0,0 +1,233 @@ +//package com.alibaba.middleware.race.mom; +// +//import io.netty.bootstrap.Bootstrap; +//import io.netty.channel.ChannelFuture; +//import io.netty.channel.ChannelHandlerContext; +//import io.netty.channel.ChannelInitializer; +//import io.netty.channel.ChannelOption; +//import io.netty.channel.SimpleChannelInboundHandler; +//import io.netty.channel.nio.NioEventLoopGroup; +//import io.netty.channel.socket.SocketChannel; +//import io.netty.channel.socket.nio.NioSocketChannel; +//import io.netty.handler.codec.LineBasedFrameDecoder; +//import io.netty.handler.codec.string.StringDecoder; +// +//import java.util.ArrayList; +//import java.util.Map; +//import java.util.concurrent.BlockingQueue; +//import java.util.concurrent.ConcurrentHashMap; +//import java.util.concurrent.ExecutorService; +//import java.util.concurrent.Executors; +//import java.util.concurrent.LinkedBlockingQueue; +//import java.util.concurrent.TimeUnit; +//import java.util.concurrent.atomic.AtomicLong; +// +//import org.slf4j.Logger; +//import org.slf4j.LoggerFactory; +// +//import com.alibaba.middleware.race.mom.serializer.RpcEncoder; +//import com.alibaba.middleware.race.mom.util.InfoBodyProduceList; +// +// +//public class DefaultProducer1 implements Producer { +// transient private static Logger logger = LoggerFactory.getLogger(DefaultProducer.class); +// private String topic; +// private String groupId; +// private static String producerId="pro_"+String.valueOf(Math.random()*100); +// private ChannelFuture future; +// private NioEventLoopGroup group; +// private String SIP; +// Object obj=new Object(); +// private SendCallback callback; +// private SendResult sr; +// private Map> resultMap=new ConcurrentHashMap>(); +// private BlockingQueue sendQueue= new LinkedBlockingQueue(100); +// private ExecutorService sendPool = Executors.newFixedThreadPool(1); +// private Bootstrap bootstrap; +// public DefaultProducer1() {startSend();} +// +// @Override +// public void asyncSendMessage(Message message, SendCallback callback) { +// MessageInfo msgInfo = buildMessage(message, callback); +// //加入同步等待队列 +// logger.debug("生产者准备发起请求和消息"); +// resultMap.put(message.getMsgId(), new LinkedBlockingQueue(1)); +// future.channel().writeAndFlush(msgInfo); +// } +// /** +// * 对Message 进行必要的信息设置,测试设置了groupId和topic producer.setGroupId(PID+code);producer.setTopic(topic); +// * @param msg +// * @param callback +// * @return +// */ +// private MessageInfo buildMessage (Message msg, SendCallback callback) { +// if (callback != null) { +// this.callback = callback; +// logger.debug("生产者异步发送消息"); +// } +// +// msg.setTopic(topic); +// msg.setBornTime(System.currentTimeMillis()); +// msg.makeID(); +// return new MessageInfo(msg,producerId,this.groupId); +// } +// +// @Override +// public SendResult sendMessage(Message message) { +// MessageInfo msgInfo = buildMessage(message, null); +// while(!sendQueue.offer(e)){ +// +// } +// +// if(sendQueue.put(msgInfo);(msgInfo)){ +// sendQueue.size() +// }else { +// +// new MessageInfoQueue(sendQueue,); +// sendQueue=new LinkedBlockingQueue(100); +// } +// +// synchronized(obj){ +// try { +// obj.wait(); +// } catch (InterruptedException e) { +// } +//// resultMap.put(message.getMsgId(), new LinkedBlockingQueue(1)); +//// SendResult sr=null; +//// try { +//// sr=resultMap.get(message.getMsgId()).take(); +//// } catch (InterruptedException e) { +//// sr=new SendResult(); +//// sr.setStatus(SendStatus.FAIL); +//// } +// +// return sr;} +// } +// +// public void buildQueue(){ +// MessageInfoQueue msgInfoQueue=new MessageInfoQueue(); +// while(!sendQueue.isEmpty()){ +// sendQueue. +// msgInfoQueue. +// } +// Message msg=sendQueue.poll(); +// +// +// +// +// } +// +// +// @Override +// public void setGroupId(String groupId) { +// // TODO Auto-generated method stub +// this.groupId = groupId; +// } +// +// @Override +// public void setSIP(String sip) { +// this.SIP = sip; +// } +// +// @Override +// public void setTopic(String topic) { +// this.topic = topic; +// } +// public void startSend(){ +// sendPool.execute(new Runnable(){ +// +// @Override +// public void run() { +// while(true){ +// System.out.println(sendQueue.size()); +// if(sendQueue.size()==200){ +// future.channel().writeAndFlush(sendQueue); +// sendQueue.clear(); +// } +// } +// } +// +// }); +// } +// @Override +// public void start() { +//// try { +//// Thread.sleep(500); +//// } catch (InterruptedException e1) { +//// e1.printStackTrace(); +//// } +// SIP=System.getProperty("SIP"); +// if(SIP==null) +// SIP="127.0.0.1"; +// System.out.println("connect:"+System.getProperty("SIP")); +// group = new NioEventLoopGroup(); +// try { +// bootstrap = new Bootstrap(); +// bootstrap.group(group).channel(NioSocketChannel.class) +// .option(ChannelOption.SO_REUSEADDR, true).option(ChannelOption.SO_KEEPALIVE, true) +// .handler(new ChannelInitializer() { +// @Override +// public void initChannel(SocketChannel channel) throws Exception { +// channel.pipeline() +// .addLast(new RpcEncoder()) +// .addLast(new LineBasedFrameDecoder(1024)) +// .addLast(new StringDecoder()) +// .addLast(new SimpleChannelInboundHandler() { +// +// @Override +// public void channelInactive(ChannelHandlerContext ctx) throws Exception { +// logger.error("producer失去broker链接"); +// connect(); +// } +// @Override +// protected void channelRead0(ChannelHandlerContext arg0, Object info) throws Exception { +// // main work; +// String msgId=info.toString(); +// sr=new SendResult(); +// sr.setMsgId(msgId); +// synchronized(obj){ +//// DefaultProducer.this.sr=sr; +// obj.notifyAll(); +//// obj.notify(); +// } +//// if (null == callback) { +//// resultMap.get(msgId).add(sr); +//// return; +//// } else { +//// // 异步方式接受数据 +//// callback.onResult(sr); +//// } +// } +// @Override +// public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { +// +// logger.error("--生产者的异常关闭--" + cause.getMessage()); +// cause.printStackTrace(); +// ctx.close(); +// } +// }); +// } +// }); +// connect(); +// } catch (InterruptedException e) { +// logger.error("producer 抛出异常 " + e.getMessage()); +// } +// } +// public void connect() throws InterruptedException +// { +// try{ +// future = bootstrap.connect(SIP, 9999).sync(); +// logger.error("连接成功"); +// }catch(Exception e) +// { +// Thread.sleep(1000); +// logger.error("重新连接"); +// connect(); +// } +// } +// @Override +// public void stop() { +// group.shutdownGracefully(); +// } +// +//} diff --git a/src/main/java/com/alibaba/middleware/race/mom/DefaultProducer2.java b/src/main/java/com/alibaba/middleware/race/mom/DefaultProducer2.java new file mode 100644 index 0000000..b733fba --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/DefaultProducer2.java @@ -0,0 +1,191 @@ +//package com.alibaba.middleware.race.mom; +// +//import io.netty.bootstrap.Bootstrap; +//import io.netty.channel.ChannelFuture; +//import io.netty.channel.ChannelHandlerContext; +//import io.netty.channel.ChannelInitializer; +//import io.netty.channel.ChannelOption; +//import io.netty.channel.SimpleChannelInboundHandler; +//import io.netty.channel.nio.NioEventLoopGroup; +//import io.netty.channel.socket.SocketChannel; +//import io.netty.channel.socket.nio.NioSocketChannel; +//import io.netty.handler.codec.LineBasedFrameDecoder; +//import io.netty.handler.codec.string.StringDecoder; +// +//import java.util.ArrayList; +//import java.util.Map; +//import java.util.concurrent.BlockingQueue; +//import java.util.concurrent.ConcurrentHashMap; +//import java.util.concurrent.ExecutorService; +//import java.util.concurrent.Executors; +//import java.util.concurrent.LinkedBlockingQueue; +//import java.util.concurrent.TimeUnit; +//import java.util.concurrent.atomic.AtomicLong; +// +//import org.slf4j.Logger; +//import org.slf4j.LoggerFactory; +// +//import com.alibaba.middleware.race.mom.serializer.RpcEncoder; +//import com.alibaba.middleware.race.mom.util.InfoBodyProduceList; +// +// +//public class DefaultProducer2 implements Producer { +// transient private static Logger logger = LoggerFactory.getLogger(DefaultProducer.class); +// private String topic; +// private String groupId; +// private static String producerId="pro_"+String.valueOf(Math.random()*100); +// private ChannelFuture future; +// private NioEventLoopGroup group; +// private String SIP; +// private SendCallback callback; +// private Map> resultMap=new ConcurrentHashMap>(); +// private Bootstrap bootstrap; +// Object obj=new Object(); +// private SendResult sendResult; +// public DefaultProducer2() {} +// +// @Override +// public void asyncSendMessage(Message message, SendCallback callback) { +// MessageInfo msgInfo = buildMessage(message, callback); +// //加入同步等待队列 +// logger.debug("生产者准备发起请求和消息"); +// resultMap.put(message.getMsgId(), new LinkedBlockingQueue(1)); +// future.channel().writeAndFlush(msgInfo); +// } +// /** +// * 对Message 进行必要的信息设置,测试设置了groupId和topic producer.setGroupId(PID+code);producer.setTopic(topic); +// * @param msg +// * @param callback +// * @return +// */ +// private MessageInfo buildMessage (Message msg, SendCallback callback) { +// if (callback != null) { +// this.callback = callback; +// logger.debug("生产者异步发送消息"); +// } +// +// msg.setTopic(topic); +// msg.setBornTime(System.currentTimeMillis()); +// msg.makeID(); +// return new MessageInfo(msg,producerId,this.groupId); +// } +// +// @Override +// public SendResult sendMessage(Message message) { +// MessageInfo msgInfo = buildMessage(message, null); +// //resultMap.put(msg.getMsgId(), new LinkedBlockingQueue(1)); +// future.channel().writeAndFlush(msgInfo); +// //SendResult sr=null; +// synchronized(obj){ +// try { +// obj.wait(); +// } catch (InterruptedException e) { +// } +// } +//// try { +//// sr=resultMap.get(msg.getMsgId()).take(); +//// } catch (InterruptedException e) { +//// sr=new SendResult(); +//// sr.setStatus(SendStatus.FAIL); +//// } +// return sendResult; +// } +// +// @Override +// public void setGroupId(String groupId) { +// // TODO Auto-generated method stub +// this.groupId = groupId; +// } +// +// @Override +// public void setSIP(String sip) { +// this.SIP = sip; +// } +// +// @Override +// public void setTopic(String topic) { +// this.topic = topic; +// } +// +// @Override +// public void start() { +//// try { +//// Thread.sleep(500); +//// } catch (InterruptedException e1) { +//// e1.printStackTrace(); +//// } +// SIP=System.getProperty("SIP"); +// if(SIP==null) +// SIP="127.0.0.1"; +// System.out.println("connect:"+System.getProperty("SIP")); +// group = new NioEventLoopGroup(); +// try { +// bootstrap = new Bootstrap(); +// bootstrap.group(group).channel(NioSocketChannel.class) +// .option(ChannelOption.SO_REUSEADDR, true).option(ChannelOption.SO_KEEPALIVE, true) +// .handler(new ChannelInitializer() { +// @Override +// public void initChannel(SocketChannel channel) throws Exception { +// channel.pipeline() +// .addLast(new RpcEncoder()) +// .addLast(new LineBasedFrameDecoder(1024)) +// .addLast(new StringDecoder()) +// .addLast(new SimpleChannelInboundHandler() { +// +// @Override +// public void channelInactive(ChannelHandlerContext ctx) throws Exception { +// logger.error("producer失去broker链接"); +// connect(); +// } +// @Override +// protected void channelRead0(ChannelHandlerContext arg0, Object info) throws Exception { +// // main work; +// String msgId=info.toString(); +// SendResult sr=new SendResult(); +// sendResult=sr; +// sr.setMsgId(msgId); +// if (null == callback) { +// synchronized(obj){ +//// DefaultProducer.this.sr=sr; +//// obj.notifyAll(); +// obj.notify(); +// } +// //resultMap.get(result).put(sr); +// } else { +// // 异步方式接受数据 +// callback.onResult(sr); +// } +// } +// @Override +// public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { +// +// logger.error("--生产者的异常关闭--" + cause.getMessage()); +// cause.printStackTrace(); +// ctx.close(); +// } +// }); +// } +// }); +// connect(); +// } catch (InterruptedException e) { +// logger.error("producer 抛出异常 " + e.getMessage()); +// } +// } +// public void connect() throws InterruptedException +// { +// try{ +// future = bootstrap.connect(SIP, 9999).sync(); +// logger.error("连接成功"); +// }catch(Exception e) +// { +// Thread.sleep(1000); +// logger.error("重新连接"); +// connect(); +// } +// } +// @Override +// public void stop() { +// group.shutdownGracefully(); +// } +// +//} diff --git a/src/main/java/com/alibaba/middleware/race/mom/Message.java b/src/main/java/com/alibaba/middleware/race/mom/Message.java new file mode 100644 index 0000000..33d159a --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/Message.java @@ -0,0 +1,82 @@ +package com.alibaba.middleware.race.mom; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Message ,目前只需要 三个属性,用于消费者消费(可能有遗漏,根据消费者消费即可) + * @author youngforever + */ +public class Message implements Serializable{ + + private static final long serialVersionUID = 1982349723732590L; + private transient static AtomicLong msgIdProduce=new AtomicLong(); + /** + * 测试严证,消费等需要的必要字段 + */ + private String msgId; + private String topic; + private byte[] body; + private Map properties = new HashMap(); + private long bornTime; + + +// public void makeID(){ +// if(msgId==null){ +// msgId=this.topic+this.properties.toString()+"_"+String.valueOf(msgIdProduce.incrementAndGet()); +// } +// } + + public String getTopic() { + return topic; + } + public Message(){} + public Message( String topic, byte[] body, + Map properties, long bornTime) { + this.topic = topic; + this.body = body; + this.properties = properties; + this.bornTime = bornTime; + } + public void setTopic(String topic) { + this.topic = topic; + } + public byte[] getBody() { + return body; + } + public void setBody(byte[] body) { + this.body = body; + } + public Map getProperties() { + return properties; + } + public void setProperties(Map properties) { + this.properties = properties; + } + public void setProperty(String key,String property){ + this.properties.put(key, property); + } + public String getProperty(String key){ + return this.properties.get(key); + } + public long getBornTime() { + return bornTime; + } + public void setBornTime(long bornTime) { + this.bornTime = bornTime; + } + + public String getMsgId() { + return msgId; + } + + public void setMsgId(String msgId) { + this.msgId = msgId; + } + public String toString(){ + return this.msgId; + } + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/MessageListener.java b/src/main/java/com/alibaba/middleware/race/mom/MessageListener.java new file mode 100644 index 0000000..fa54e9b --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/MessageListener.java @@ -0,0 +1,11 @@ +package com.alibaba.middleware.race.mom; + + +public interface MessageListener { + /** + * + * @param message + * @return + */ + ConsumeResult onMessage(Message message); +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/Producer.java b/src/main/java/com/alibaba/middleware/race/mom/Producer.java new file mode 100644 index 0000000..259369f --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/Producer.java @@ -0,0 +1,20 @@ +package com.alibaba.middleware.race.mom; + + +public interface Producer { + + void setSIP(String sip); + + void start(); + + void setTopic(String topic); + + void setGroupId(String groupId); + + SendResult sendMessage(Message message); + + void asyncSendMessage(Message message, SendCallback callback); + + void stop(); + +} \ No newline at end of file diff --git a/src/main/java/com/alibaba/middleware/race/mom/QueueMessage.java b/src/main/java/com/alibaba/middleware/race/mom/QueueMessage.java new file mode 100644 index 0000000..c0d7259 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/QueueMessage.java @@ -0,0 +1,6 @@ +package com.alibaba.middleware.race.mom; + +public class QueueMessage extends Message{ + private byte[] body; + private long bornTime; +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/SendCallback.java b/src/main/java/com/alibaba/middleware/race/mom/SendCallback.java new file mode 100644 index 0000000..8b389ee --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/SendCallback.java @@ -0,0 +1,5 @@ +package com.alibaba.middleware.race.mom; + +public interface SendCallback { + void onResult(SendResult result); +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/SendResult.java b/src/main/java/com/alibaba/middleware/race/mom/SendResult.java new file mode 100644 index 0000000..5913fe5 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/SendResult.java @@ -0,0 +1,36 @@ +package com.alibaba.middleware.race.mom; + +import java.io.Serializable; + +public class SendResult implements Serializable { + /** + * + */ + private static final long serialVersionUID = 321275686710864167L; + private String info; + private SendStatus status=SendStatus.SUCCESS; + private String msgId; + public String getInfo() { + return info; + } + public String getMsgId() { + return msgId; + } + public SendStatus getStatus() { + return status; + } + public void setInfo(String info) { + this.info = info; + } + public void setMsgId(String msgId) { + this.msgId = msgId; + } + public void setStatus(SendStatus status) { + this.status = status; + } + @Override + public String toString(){ + return "msg "+msgId+" send "+(status==SendStatus.SUCCESS?"success":"fail")+" info:"+info; + } + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/SendStatus.java b/src/main/java/com/alibaba/middleware/race/mom/SendStatus.java new file mode 100644 index 0000000..aa6a7b5 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/SendStatus.java @@ -0,0 +1,30 @@ +package com.alibaba.middleware.race.mom; +/** + * 发送状态 + * + */ +public enum SendStatus{ + SUCCESS, + FAIL, + TIMEOUT/*收到ack超时*/, + /** + * 投递中 + */ + SEND, + /** + * 重投中 + */ + RESEND, + /** + * 消费失败重投 + */ + FAILRESEND, + /** + * 超时重投 + */ + TIMEOUTRESEND, + /** + * 订阅集群所有消费者不在线 + */ + GROUPLOST +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/broker/function/AccumulateHandler.java b/src/main/java/com/alibaba/middleware/race/mom/broker/function/AccumulateHandler.java new file mode 100644 index 0000000..6f497e0 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/broker/function/AccumulateHandler.java @@ -0,0 +1,32 @@ +package com.alibaba.middleware.race.mom.broker.function; + +import java.util.Timer; +import java.util.TimerTask; + +import com.alibaba.middleware.race.mom.config.Config; + +/** + * 消息堆积处理,当内存消息堆积过多,将不再内存中缓存消息,堆积减少后从硬盘拉取消息 + * 如果有大量超时消费者集群和正常快速消费集群,大量超时消息将不堆积内存,从磁盘拉取,保证不影响正常集群的消费tps + * @author young + * + */ +public class AccumulateHandler { + public static volatile boolean full =false; + private static Timer memoryCheckTimer=new Timer(); + /** + * 是否可以继续内存添加消息 + * @param memoryCacheMaxOffset + * @return + */ + public static void startCheckMemory(){ + memoryCheckTimer.schedule(new TimerTask(){ + @Override + public void run() { + if((double)Runtime.getRuntime().freeMemory()/Runtime.getRuntime().totalMemory()<(1-Config.memoryFullFactor)){//可用内存小于虚拟机内存3/4时不能再往内存堆积消息 + full=true; + } + } + },5000,5000); + } +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/broker/function/MessageInfo.java b/src/main/java/com/alibaba/middleware/race/mom/broker/function/MessageInfo.java new file mode 100644 index 0000000..6c14e3f --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/broker/function/MessageInfo.java @@ -0,0 +1,134 @@ +package com.alibaba.middleware.race.mom.broker.function; +import io.netty.channel.Channel; + +import java.io.Serializable; + +import com.alibaba.middleware.race.mom.Message; + +/** + * 生产者生的产消息和其它信息封装 + * @author youngforever + * + */ +public class MessageInfo implements Serializable{ + private static final long serialVersionUID = 529580833582395809L; + /** + * 由broker端设置,需要再加 + */ + private Message msg; +// private String producerId; +// private String producerGroupId; + private String msgInfoId; + private Channel producer; + private int queueId; + private int offset;//存储顺序,是topicandfilter下 某个queueId 下的offset + + + + public MessageInfo(Message msg, Channel producer){ + this.msg = msg; + this.producer = producer; + } + public MessageInfo(Message msg, Channel producer, int queueId, int offset) { + this.msg = msg; + this.producer = producer; + this.queueId = queueId; + this.offset = offset; + } + + + /** + * msg 订阅集群数 + */ +// private AtomicInteger subGroupsCount; + +// public MessageInfo(Message msg, String producerId, String producerGroupId) { +// this.msg = msg; +// this.producerId = producerId; +// this.producerGroupId = producerGroupId; +// } +// + public Message getMsg() { + return msg; + } + + + public void setMsg(Message msg) { + this.msg = msg; + } + + +// public int getSubGroupsCount() { +// return subGroupsCount.get(); +// } +// +// public void setSubGroupsCount(int subGroupsCount) { +// this.subGroupsCount = new AtomicInteger(subGroupsCount); +// } +// public int decreSubGroupsCount(){ +// return subGroupsCount.decrementAndGet(); +// } +// public String getProducerId() { +// return producerId; +// } + + + public String getMsgInfoId() { + return msgInfoId; + } + public void setMsgInfoId(String msgInfoId) { + this.msgInfoId = msgInfoId; + } + public int getQueueId() { + return queueId; + } + public int getQueueIndex() { + return queueId; + } + + + public void setQueueId(int queueId) { + this.queueId = queueId; + } + + +// public void setProducerId(String producerId) { +// this.producerId = producerId; +// } + + + public int getOffset() { + return offset; + } + + + public void setOffset(int offset) { + this.offset = offset; + } + + +// public String getProducerGroupId() { +// return producerGroupId; +// } +// +// +// public void setProducerGroupId(String producerGroupId) { +// this.producerGroupId = producerGroupId; +// } + + + public Channel getProducer() { + return producer; + } + + + /** + *由broker 端设置,连接成功后,设置生产者channel,以便精确返回ack + * @param producer + */ + public void setProducer(Channel producer) { + this.producer = producer; + } + + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/broker/function/MessageInfoQueue.java b/src/main/java/com/alibaba/middleware/race/mom/broker/function/MessageInfoQueue.java new file mode 100644 index 0000000..b27f850 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/broker/function/MessageInfoQueue.java @@ -0,0 +1,31 @@ +package com.alibaba.middleware.race.mom.broker.function; + +import io.netty.channel.Channel; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.BlockingQueue; + +import com.alibaba.middleware.race.mom.QueueMessage; + +public class MessageInfoQueue { + private Channel producer; + private BlockingQueue msgQueue; + private String topic; + private Map properties = new HashMap(); + private String msgQueueId; + + + public MessageInfoQueue(BlockingQueue msgQueue, String topic, + Map properties) { + super(); + this.msgQueue = msgQueue; + this.topic = topic; + this.properties = properties; + } + + + + + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/broker/function/MessageManager.java b/src/main/java/com/alibaba/middleware/race/mom/broker/function/MessageManager.java new file mode 100644 index 0000000..c7b4005 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/broker/function/MessageManager.java @@ -0,0 +1,344 @@ +package com.alibaba.middleware.race.mom.broker.function; + + +import java.util.ArrayList; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import io.netty.channel.ChannelHandlerContext; + +import org.slf4j.LoggerFactory; + +import com.alibaba.middleware.race.mom.Message; +import com.alibaba.middleware.race.mom.broker.group.ConsumerGroup; +import com.alibaba.middleware.race.mom.broker.group.ProducerGroup; +import com.alibaba.middleware.race.mom.util.TopicAndFilter; + +public class MessageManager { + private static org.slf4j.Logger logger = LoggerFactory.getLogger(MessageManager.class); + +// public static Map messageCacheMap; + /** + * 每个集群有独立的消费进度 + * 消息再broker收到后,按照TopicAndFilter 消息分类,每种TopicAndFilter消息有独立的offsetProducer + * 如果要以queueId分多文件,那么再分一层独立的offsetProducer + */ + +// private MsgStore mstore = /* new MsgStoreImp();// */new MsgStoreImp_MappedBuffer(); +// private FSTConfiguration fst = FSTConfiguration.getDefaultConfiguration(); + +// private Map/*reSendMsgQueue*/>topicAndgroup2reSendMsgQueue=new HashMap<>(); + + + +// public MessageManager(ConcurrentHashMap recoverMsgMap){ +// if(recoverMsgMap!=null){ +// messageCacheMap=recoverMsgMap; +// }else{ +// messageCacheMap=new ConcurrentHashMap<>(); +// } +// } + + + /** + * 打包消息处理 + * @param msgSend + * @param producer + */ + public static void recieveMsg(MessageSend msgSend,ChannelHandlerContext producer){ + long start=System.currentTimeMillis(); + /* + * 存储处理 + */ + msgSend.setProducer(producer.channel()); + TopicAndFilter topicAndFilter=new TopicAndFilter(msgSend.getTopic(),msgSend.getProperties()); + String queueIdAndOffsetArray[] = ProducerGroup.storeMsg(topicAndFilter,msgSend).split(" "); + int queueIndex=Integer.parseInt(queueIdAndOffsetArray[0]); + int offset=Integer.parseInt(queueIdAndOffsetArray[1]); + logger.debug("currentCanUseOffset "+offset); + /* + * 消息进入待发送队列，等待存储完成（） + */ + int i=0; + for(byte[]body:msgSend.getBodys()){ + Message msg=new Message(msgSend.getTopic(),body,msgSend.getProperties(),msgSend.getBornTime()); + msg.setMsgId(msgSend.getSendIds().get(i)); + MessageInfo msgInfo=new MessageInfo(msg,producer.channel()); + msgInfo.setQueueId(queueIndex); + msgInfo.setOffset(offset++); + msgInfo.setMsgInfoId(topicAndFilter.toString()+queueIndex+""+String.valueOf(msgInfo.getOffset())); + /* + * 发送处理，加入发送队列，等待存储完成 + */ + ConsumerGroup.sendMsg(topicAndFilter, msgInfo); + i++; + } +// logger.error("消息born 到生产者recv处理完毕 cost："+(System.currentTimeMillis()-msgSend.getBornTime())); +// logger.error("消息recv生产者recv处理完毕 cost："+(System.currentTimeMillis()-start)); + } + + + /** + * 接收到生产者 消息,触发一系列处理 + * @param msg + * @param producer + */ +// public static void recieveMsg(Message msg,ChannelHandlerContext producer){ +// //生产者信息构建的topicAndFilter +// TopicAndFilter topicAndFilter=new TopicAndFilter(msg.getTopic(),msg.getProperties()); +//// String topicAndFilterString=topicAndFilter.toString(); +// /** +// * broker端给消息的编号:offset,同时对应存储位置,消费进度 +// * topicAndFilter不同,offset将相互独立 +// */ +//// //broker端给消息的编号:offset,同时对应存储位置,消费进度 +//// if(offsetProducerMap.containsKey(topicAndFilterString)){ +//// int offset=offsetProducerMap.get(topicAndFilterString).getAndIncrement(); +//// msg.setOffset(offset); +//// } +// /** +// * 以下两个过程,保证: +// * 一.先加入持久化队列(以持久化队列下标编offset(对应消息在文件中的索引),保证正确对应(也不会受多线程影响)),再加入发送队列 +// * 二.保证每一条消息到磁盘(存储底层也已经强行force),再返回ack +// * 三.发送队列与持久化队列一一对应(其实也可以允许队列均衡导致不一致,但大的来说顺序仍然一致),保证消费进度的正确性(我们可以通过配置,每一个topicandfilter下做几个队列) +// * 四.发送给消费者与消息存储,多线程执行,不保证先后顺序(理想是并行) +// */ +// long start=System.currentTimeMillis(); +// MessageInfo msgInfo=new MessageInfo(msg,producer.channel()); +// //持久化消息,成功后返回给生产者ack +// String queueIdAndOffsetArray[]=ProducerGroup.storeMsg(topicAndFilter,msgInfo).split(" "); +// int queueIndex=Integer.parseInt(queueIdAndOffsetArray[0]); +// int offset=Integer.parseInt(queueIdAndOffsetArray[1]); +// msgInfo.setQueueId(queueIndex); +// msgInfo.setOffset(offset); +// //消息对象引用存入内存map,为了避免多topicAndFilter时,map key重复,key以topicAndFilter+offset +//// msg.setOffset(offset); +//// MessageManager.addMessage(topicAndFilterString+String.valueOf(offset),msgInfo); +// //转发给发送消费者 +// ConsumerGroup.sendMsg(topicAndFilter, msgInfo); +// +//// mstore.writeByteNormal(topicAndfilter.toString(),"t","t",0+"" , msgbytes/*caches*/); +//// ProducerGroup.addProducer(producer.channel(),"PROO", topicAndFilter); +//// producer.channel().writeAndFlush(msg.getMsgId() + "\r\n"); +// //接收新消息处理完成,等待消费结果,进入消费结果处理 +// logger.error("消息born 到生产者recv处理完毕 cost："+(System.currentTimeMillis()-msg.getBornTime())); +// logger.error("消息recv生产者recv处理完毕 cost："+(System.currentTimeMillis()-start)); +// } + + + +// public static void recieveMsg(BlockingQueue msgInfoqueue,ChannelHandlerContext producer){ +// MessageInfo msgInfo=msgInfoqueue.peek(); +// //生产者信息构建的topicAndFilter +// TopicAndFilter topicAndFilter=new TopicAndFilter(msgInfo.getMsg().getTopic(),msgInfo.getMsg().getProperties()); +// String topicAndFilterString=topicAndFilter.toString(); +// /** +// * broker端给消息的编号:offset,同时对应存储位置,消费进度 +// * topicAndFilter不同,offset将相互独立 +// */ +//// //broker端给消息的编号:offset,同时对应存储位置,消费进度 +//// if(offsetProducerMap.containsKey(topicAndFilterString)){ +//// int offset=offsetProducerMap.get(topicAndFilterString).getAndIncrement(); +//// msg.setOffset(offset); +//// } +// /** +// * 以下两个过程,保证: +// * 一.先加入持久化队列(以持久化队列下标编offset(对应消息在文件中的索引),保证正确对应(也不会受多线程影响)),再加入发送队列 +// * 二.保证每一条消息到磁盘(存储底层也已经强行force),再返回ack +// * 三.发送队列与持久化队列一一对应(其实也可以允许队列均衡导致不一致,但大的来说顺序仍然一致),保证消费进度的正确性(我们可以通过配置,每一个topicandfilter下做几个队列) +// * 四.发送给消费者与消息存储,多线程执行,不保证先后顺序(理想是并行) +// */ +// msgInfo.setProducer(producer.channel()); +// MessageInfoQueue msgQueue=new MessageInfoQueue(producer.channel(),topicAndFilter,msgInfoqueue); +// +// //持久化消息,成功后返回给生产者ack +// ProducerGroup.storeMsg(topicAndFilter,msgQueue); +//// int queueIndex=Integer.parseInt(queueIdAndOffsetArray[0]); +//// int offset=Integer.parseInt(queueIdAndOffsetArray[1]); +//// msgInfo.setQueueId(queueIndex); +//// msgInfo.setOffset(offset); +// //消息对象引用存入内存map,为了避免多topicAndFilter时,map key重复,key以topicAndFilter+offset +//// msg.setOffset(offset); +//// MessageManager.addMessage(topicAndFilterString+String.valueOf(offset),msgInfo); +// //转发给发送消费者 +// ConsumerGroup.sendMsg(topicAndFilter, msgInfo); +// +//// mstore.writeByteNormal(topicAndfilter.toString(),"t","t",0+"" , msgbytes/*caches*/); +//// ProducerGroup.addProducer(producer.channel(),"PROO", topicAndFilter); +//// producer.channel().writeAndFlush(msg.getMsgId() + "\r\n"); +// //接收新消息处理完成,等待消费结果,进入消费结果处理 +// } + +// public static void addMessage(String offet,MessageInfo msgInfo){ +// messageCacheMap.put(offet, msgInfo); +// } +// public static void setMsgMap(ConcurrentHashMap recoverMap){ +// messageCacheMap=recoverMap; +// } + + + + /** + * 开始存储任务 + */ +// long starttttt; +// public void startStore(){ +// +//// for(final BlockingQueue storeQueue:storeQueues){ +//// for(int i=0;i storeQueue=storeQueues.get(0); +// while(true){ +// MessageInfo msgInfo=null; +// ArrayList msgbytes=new ArrayList<>(); +// StoreSucceedListener storeSucceedListener=new StoreSucceedListener(); +// /** +// * 准备一次存储过程,构建对应存储成功监听器 +// */ +//// int i=100; +//// try { +//// count.await(20, TimeUnit.MILLISECONDS); +//// } catch (InterruptedException e) { +//// count=new CountDownLatch(100); +//// } +//// boolean flag=false; +//// while(!storeQueue.isEmpty()&&i-->0){ +// +// int offset=0; +// +// while(true){ +//// if(storeCount.incrementAndGet()>98){ +//// System.out.println("一百条等待"+(System.currentTimeMillis()-start)); +//// break; +//// }; +// try { +// msgInfo=storeQueue.poll(0, TimeUnit.MILLISECONDS); +// } catch (InterruptedException e) { +// // TODO Auto-generated catch block +// e.printStackTrace(); +// } +// if(msgInfo==null&&storeCount.get()==0) continue; +// if(msgInfo==null) continue; +// offset=msgInfo.getOffset(); +// redo.writeLog2(offset+"", msgInfo.getMsg().getTopic(), msgInfo.getQueueId(),msgInfo.getMsg().getBody()); +//// i++; +// +//// storeSucceedListener.addAckWaiter(msgInfo.getProducer(), msgInfo.getMsg().getMsgId()); +// storeSucceedListener.addAckWaiterGroup(msgInfo.getProducer(), msgInfo.getMsg().getMsgId()); +// +//// if(cando&&storeCount.get()!=0) break; +// if(storeCount.incrementAndGet()>199) break; +//// flag=true; +//// +// } +//// if(flag) +// if(storeCount.get()==0) continue; +// logger.error(Thread.currentThread().getName()+" "+(System.currentTimeMillis()-start)+" ms后提交新落盘任务,一次flush"+storeCount.get()+"条"); +//// storeTask(storeQueues.indexOf(storeQueue),msgbytes,storeSucceedListener,offset); +// +// long submit=System.currentTimeMillis(); +// if(starttttt==0){ +// starttttt=submit; +// } +// redo.flushLog(); +// logger.error("redo"+(System.currentTimeMillis()-starttttt)); +// storeSucceedListener.onStoreSucceed(topicAndfilter,0,offset); +// storeCount.set(0); +// start =System.currentTimeMillis(); +//// }else{ +//// //0就不提交了，大量废任务进入线程池。。。 +//// } +// } +// } +// }); +//// } + + + + + +//// for(int i=0;i storeQueue=storeQueues.get(1); +// while(true){ +// MessageInfo msgInfo=null; +// ArrayList msgbytes=new ArrayList<>(); +// StoreSucceedListener storeSucceedListener=new StoreSucceedListener(); +// /** +// * 准备一次存储过程,构建对应存储成功监听器 +// */ +//// int i=100; +//// try { +//// count.await(20, TimeUnit.MILLISECONDS); +//// } catch (InterruptedException e) { +//// count=new CountDownLatch(100); +//// } +//// boolean flag=false; +//// while(!storeQueue.isEmpty()&&i-->0){ +// +// int offset=0; +// +// while(true){ +//// if(storeCount.incrementAndGet()>98){ +//// System.out.println("一百条等待"+(System.currentTimeMillis()-start)); +//// break; +//// }; +// try { +// msgInfo=storeQueue.poll(0, TimeUnit.MILLISECONDS); +// } catch (InterruptedException e) { +// // TODO Auto-generated catch block +// e.printStackTrace(); +// } +// if(msgInfo==null&&storeCount.get()==0) continue; +// if(msgInfo==null) continue; +// offset=msgInfo.getOffset(); +// redo2.writeLog2(offset+"", msgInfo.getMsg().getTopic(), msgInfo.getQueueId(),msgInfo.getMsg().getBody()); +//// i++; +// +//// storeSucceedListener.addAckWaiter(msgInfo.getProducer(), msgInfo.getMsg().getMsgId()); +// storeSucceedListener.addAckWaiterGroup(msgInfo.getProducer(), msgInfo.getMsg().getMsgId()); +// +//// if(cando&&storeCount.get()!=0) break; +// if(storeCount.incrementAndGet()>99) break; +//// flag=true; +//// +// } +//// if(flag) +// if(storeCount.get()==0) continue; +// logger.error(Thread.currentThread().getName()+" "+(System.currentTimeMillis()-start)+" ms后提交新落盘任务,一次flush"+storeCount.get()+"条"); +//// storeTask(storeQueues.indexOf(storeQueue),msgbytes,storeSucceedListener,offset); +// long submit=System.currentTimeMillis(); +// if(starttttt==0){ +// starttttt=submit; +// } +// redo2.flushLog(); +// logger.error("redo2 "+(System.currentTimeMillis()-starttttt)); +// storeSucceedListener.onStoreSucceed(topicAndfilter,1,offset); +// storeCount.set(0); +// start =System.currentTimeMillis(); +//// }else{ +//// //0就不提交了，大量废任务进入线程池。。。 +//// } +// } +// } +// }); +//// } +// +// + + + +// } +// } + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/broker/function/MessageSend.java b/src/main/java/com/alibaba/middleware/race/mom/broker/function/MessageSend.java new file mode 100644 index 0000000..7586fcd --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/broker/function/MessageSend.java @@ -0,0 +1,91 @@ +package com.alibaba.middleware.race.mom.broker.function; + +import io.netty.channel.Channel; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +public class MessageSend implements Serializable{ + + private static final long serialVersionUID = 1980989080980980L; + private ArrayList sendIds=new ArrayList<>(); + private String topic; + private ArrayList bodys=new ArrayList<>(); + private Map properties = new HashMap(); + private long bornTime=System.currentTimeMillis(); + private Channel producer; + + + public MessageSend(){} + + + + + public ArrayList getSendIds() { + return sendIds; + } + + + + + public void setSendIds(ArrayList sendIds) { + this.sendIds = sendIds; + } + + + + + public String getTopic() { + return topic; + } + public void setTopic(String topic) { + this.topic = topic; + } + public ArrayList getBodys() { + return bodys; + } + public void setBodys(ArrayListbodys) { + this.bodys = bodys; + } + public Map getProperties() { + return properties; + } + public void setProperties(Map properties) { + this.properties = properties; + } + public long getBornTime() { + return bornTime; + } + public void setBornTime(long bornTime) { + this.bornTime = bornTime; + } + public MessageSend( String topic, + Map properties) { + this.topic = topic; + this.properties = properties; + } + + public MessageSend( String topic + ) { + this.topic = topic; + } + + + + public Channel getProducer() { + return producer; + } + + + + public void setProducer(Channel producer) { + this.producer = producer; + } + + + + + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/broker/function/MessageStore.java b/src/main/java/com/alibaba/middleware/race/mom/broker/function/MessageStore.java new file mode 100644 index 0000000..d91e0db --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/broker/function/MessageStore.java @@ -0,0 +1,356 @@ +package com.alibaba.middleware.race.mom.broker.function; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicInteger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.middleware.race.mom.config.Config; +import com.alibaba.middleware.race.mom.store.RedoLog; +import com.alibaba.middleware.race.mom.util.TopicAndFilter; + +/** + * 每个生产者集群(组)有一个MessageStore,用于存储此集群(组)(对应一种TopicAndFilter)的消息,也即每个生产者集群(组)一个storeQueues + * @author youngforever + * + */ + +public class MessageStore { + private static Logger logger = LoggerFactory.getLogger(MessageStore.class); + + /** + * 注意这里要和sendqueuenum 一致,全局统一配置 + */ + private int storeQueueNum=Config.maxqueue; + /** + *支持分布式队列(分布式存储任务),默认一个存储队列 + */ +// private LinkedList> storeQueues = new LinkedList<>(); + private LinkedList> storeGroupQueues = new LinkedList<>(); + private int storeThreadPoolSize = storeQueueNum; + private AtomicInteger[] offsetMakers=new AtomicInteger[storeQueueNum]; + private ExecutorService storePool = Executors.newFixedThreadPool(storeThreadPoolSize); +// private ExecutorService flushPool = Executors.newSingleThreadExecutor(); + private ExecutorService flushPool = Executors.newFixedThreadPool(storeQueueNum); +// private MsgStore mstore = /* new MsgStoreImp();// */new MsgStoreImp_MappedBuffer2(); +// private FSTConfiguration fst = FSTConfiguration.getDefaultConfiguration(); + private TopicAndFilter topicAndfilter; +// private Store store=new Store(); +// Redo redo=new Redo(); +// redotest redo=new redotest(); + RedoLog storer=new RedoLog(); +// CountDownLatch count=new CountDownLatch(1); + volatile boolean cando=false; + private boolean messageGroup=false; +// private AtomicInteger recvCount=new AtomicInteger(); + + /** + * 用于恢复时,为了衔接故障前的状态 + * @param topicAndfilter + * @param queueMaxOffsets + */ + public MessageStore(TopicAndFilter topicAndfilter,Map queueMaxOffsets) { + this.topicAndfilter = topicAndfilter; + /* + * 创建时初始化队列 + */ +// for(Map.Entry queueMaxOffset:queueMaxOffsets.entrySet()){ +// this.storeQueues.add(new LinkedBlockingQueue()); +// offsetMakers[queueMaxOffset.getKey()]=new AtomicInteger(queueMaxOffset.getValue()); +// } + for(Map.Entry queueMaxOffset:queueMaxOffsets.entrySet()){ + this.storeGroupQueues.add(new LinkedBlockingQueue()); + offsetMakers[queueMaxOffset.getKey()]=new AtomicInteger(queueMaxOffset.getValue()); + } +// startStore(); + startStoreGroup(); + } + /** + * 用于新增 + * @param topicAndfilter + */ + public MessageStore(TopicAndFilter topicAndfilter) { + this.topicAndfilter = topicAndfilter; + /* + * 创建时初始化队列 + */ +// for(int i=0;i()); +// offsetMakers[i]=new AtomicInteger(); +// } + for(int i=0;i()); + offsetMakers[i]=new AtomicInteger(); + } +// startStore(); + /* + * 采用组发，则不必盯着队列 + */ +// startStoreGroup(); + } + /** + * 存储队列均衡 + * @return + */ +// private int storeQueueBalancer=-1; +// public synchronized BlockingQueue storeQueuesBalance(){ +// storeQueueBalancer++; +// if(storeQueueBalancer==storeQueueNum){ +// storeQueueBalancer=0; +// } +// BlockingQueue storeQueue=storeQueues.get(storeQueueBalancer); +// return storeQueue; +// } + private int storeGroupQueueBalancer=-1; + public synchronized BlockingQueue storeQueuesBalance(boolean isGroup){ + storeGroupQueueBalancer++; + if(storeGroupQueueBalancer==storeQueueNum){ + storeGroupQueueBalancer=0; + } + BlockingQueue storeGroupQueue=storeGroupQueues.get(storeGroupQueueBalancer); + return storeGroupQueue; + } + /** + * 加入均衡后的存储队列 + * @param msg + * @return queueId+" "+offset + */ +// public String storeMsg(MessageInfo msgInfo){ +//// System.out.println(recvCount.incrementAndGet()); +// +// BlockingQueue storeQueue=storeQueuesBalance(); +// int queueId=storeQueues.indexOf(storeQueue); +// int offset; +// synchronized(this){//同步块中进行,保证offset的准确性(存储队列中加入和offet产生的原子性) +//// count.countDown(); +// storeQueue.offer(msgInfo); +// offset=offsetMakers[queueId].getAndIncrement(); +// } +// return String.valueOf(queueId)+" "+String.valueOf(offset); +// } + /** + * 加入均衡后的组存 存储队列 + * @param msgSend + * @return + */ + public String storeMsg(MessageSend msgSend){ + BlockingQueue storeGroupQueue=storeQueuesBalance(true); + int queueId=storeGroupQueues.indexOf(storeGroupQueue); + int offset; + synchronized(this){//同步块中进行,保证offset的准确性(存储队列中加入和offet产生的原子性) +// count.countDown(); + //offset从0开始，所以返回的就是当前可用的起始offset + offset=offsetMakers[queueId].getAndAdd(msgSend.getBodys().size()); + storeGroupQueue.offer(msgSend); + } + startStoreGroup(); + return String.valueOf(queueId)+" "+String.valueOf(offset); + } +// public void storeMsg(MessageInfoQueue msgInfoQueue){ +//// System.out.println(recvCount.incrementAndGet()); +// +// BlockingQueue storeQueue=storeQueuesBalance(); +//// int queueId=storeQueues.indexOf(storeQueue); +//// int offset; +//// count.countDown(); +// storeQueue.addAll(msgInfoQueue.getMsgInfoqueue()); +//// offset=offsetMakers[queueId].getAndIncrement(); +//// } +//// return String.valueOf(queueId)+" "+String.valueOf(offset); +// } + /** + * 一次存储过程 + */ + public void storeTask(final int queueIndex,ArrayList msgbytes,final StoreSucceedListener storeSucceedListener,final int offset){ + + flushPool.execute(new Runnable(){ +// +// @Override + public void run() { + cando=false; + final long start=System.currentTimeMillis(); +// store.flush(); + storer.flushLog(); +// redo.flushLog(); +// try { +// Thread.sleep(15); +// } catch (InterruptedException e) { +// } +// logger.error("一次flush cost:"+(System.currentTimeMillis()-start)); +// mstore.writeByteNormal( topicAndfilter.toString(), String.valueOf(queueIndex), msgbytes); + cando=true; + storeSucceedListener.onStoreSucceed(topicAndfilter,queueIndex,offset); + } + + }); + + + } + + /** + * 组存任务开始 + */ + public void startStoreGroup(){ + + for(final BlockingQueue storeGroupQueue:storeGroupQueues){ + for(int i=0;i msgbytes=new ArrayList<>(); + StoreSucceedListener storeSucceedListener=new StoreSucceedListener(); + /** + * 准备一次存储过程,构建对应存储成功监听器 + */ +// int i=100; +// try { +// count.await(20, TimeUnit.MILLISECONDS); +// } catch (InterruptedException e) { +// count=new CountDownLatch(100); +// } +// boolean flag=false; +// while(!storeQueue.isEmpty()&&i-->0){ + //队列对应本次存储成功的条数 + int offset=0; + + while(!storeGroupQueue.isEmpty()){ + + msgSend=storeGroupQueue.poll(); + if(msgSend==null) break; + + storer.writeLog(msgSend.getTopic(), msgSend.getProperties().toString(),storeGroupQueues.indexOf(storeGroupQueue) , msgSend.getBodys()); + offset=msgSend.getBodys().size(); +// redo.writeLog(topic/*String*/, filter/*String*/,queueindex/*Integer*/,bodys/*(200个byte[])/*ArrayList*/); + +// for(byte[] body:msgSend.getBodys()){ +//// redo.writeLog2(body); +// offset++; +// storeCount.incrementAndGet(); +// } + +// i++; + +// storeSucceedListener.addAckWaiter(msgSend.getProducer(), msgSend.getMsg().getMsgId()); +// storeSucceedListener.addAckWaiterGroup(msgSend.getProducer(), msgSend.getMsg().getMsgId()); +// storeSucceedListener.addAckWaiterNum(msgSend.getProducer()); +// if(cando&&storeCount.get()!=0) break; +// flag=true; +// + storeSucceedListener.addAckGroupWaiter(msgSend.getProducer(),msgSend.getSendIds()); + } +// if(flag) + if(offset!=0){ +// logger.error((System.currentTimeMillis()-start)+" ms后提交新落盘任务,一次flush"+offset+"条"); +// logger.error("一共flush"+storeCount.get()*offset+"条"); + storeTask(storeGroupQueues.indexOf(storeGroupQueue),msgbytes,storeSucceedListener,offset); + storeCount.set(0); +// start =System.currentTimeMillis(); +// }else{ +// //0就不提交了，大量废任务进入线程池。。。 +// } + } + } + }); + } + } + } + + +// +// /** +// * 开始存储任务 +// */ +// public void startStore(){ +// +// for(final BlockingQueue storeQueue:storeQueues){ +// for(int i=0;i msgbytes=new ArrayList<>(); +// StoreSucceedListener storeSucceedListener=new StoreSucceedListener(); +// /** +// * 准备一次存储过程,构建对应存储成功监听器 +// */ +//// int i=100; +//// try { +//// count.await(20, TimeUnit.MILLISECONDS); +//// } catch (InterruptedException e) { +//// count=new CountDownLatch(100); +//// } +//// boolean flag=false; +//// while(!storeQueue.isEmpty()&&i-->0){ +// +// int offset=0; +// +// while(true){ +//// if(storeCount.incrementAndGet()>98){ +//// System.out.println("一百条等待"+(System.currentTimeMillis()-start)); +//// break; +//// }; +// +// try { +// if(!messageGroup){ +// msgInfo=storeQueue.poll(500, TimeUnit.MILLISECONDS); +// if(storeCount.get()>1){ +// messageGroup=true; +// } +// }else msgInfo=storeQueue.poll(); +// +// } catch (InterruptedException e) { +// // TODO Auto-generated catch block +// e.printStackTrace(); +// } +// if(msgInfo==null&&!messageGroup&&storeCount.get()==1)break; +// if(msgInfo==null&&storeCount.get()==0) continue; +// if(msgInfo==null) continue; +// offset=msgInfo.getOffset(); +// redo.writeLog(offset+"", msgInfo.getMsg().getTopic(),msgInfo.getMsg().getBody()); +//// i++; +// +//// storeSucceedListener.addAckWaiter(msgInfo.getProducer(), msgInfo.getMsg().getMsgId()); +//// storeSucceedListener.addAckWaiterGroup(msgInfo.getProducer(), msgInfo.getMsg().getMsgId()); +//// storeSucceedListener.addAckWaiterNum(msgInfo.getProducer()); +//// if(cando&&storeCount.get()!=0) break; +//// flag=true; +// if(storeCount.incrementAndGet()>199){ +// messageGroup=true; +// break; +// } +//// +// } +//// if(flag) +// if(storeCount.get()==0) continue; +// count++; +// messageGroup=true; +//// logger.error((System.currentTimeMillis()-start)+" ms后提交新落盘任务,一次flush"+storeCount.get()+"条"); +// logger.error("一共flush"+storeCount.get()*count+"条"); +// storeTask(storeQueues.indexOf(storeQueue),msgbytes,storeSucceedListener,offset); +// storeCount.set(0); +// start =System.currentTimeMillis(); +//// }else{ +//// //0就不提交了，大量废任务进入线程池。。。 +//// } +// } +// } +// }); +// } +// } +// } + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/broker/function/Offset.java b/src/main/java/com/alibaba/middleware/race/mom/broker/function/Offset.java new file mode 100644 index 0000000..a869dfa --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/broker/function/Offset.java @@ -0,0 +1,169 @@ +package com.alibaba.middleware.race.mom.broker.function; + +import java.io.Serializable; +import java.util.Map; + +import com.alibaba.middleware.race.mom.util.TopicAndFilter; + +public class Offset implements Serializable{ + private static final long serialVersionUID = -7988990980809880981L; + /** + * 消息信息,生产者集群信息 + */ + private String topic; + private Map filter; +// private TopicAndFilter topicAndFilter; +// private String groupId; + /** + * 存储采用多队列多文件时采用 + */ + private int queueIndex; + /** + * 消费者集群信息,姑且不需要,因为恢复 会重新发起订阅 + */ + + /** + * 与存储顺序(拉取索引)对应的 消费进度表示 + */ + int currentOffset=-1; //当前消费进度 + /** + * 存储成功的最大maxOffset;msg 的offset>此maxOffset,即为加入存储队列但是尚未存储(强行force)到磁盘,保证存储成功后,才来调用设置这个值 + */ + int maxOffset=-1; //最大偏移(size) + + + public Offset(){ + + }; +// public Offset(TopicAndFilter topicAndFilter){ +// this.topicAndFilter=topicAndFilter; +// } + + /*function*/ + public void addCacheMaxOffsetBy(int size) + { +// this.cacheOffset+=size; + } + public void addMaxOffsetBy(int size) + { + this.maxOffset+=size; + } + + public String getTopic() { + return topic; + } + public void setTopic(String topic) { + this.topic = topic; + } + public Map getFilter() { + return filter; + } + public void setFilter(Map filter) { + this.filter = filter; + } + public void setCurrentOffsetBy(int size) + { + if(size>this.currentOffset) + this.currentOffset=size; + } +// public String getGroupId() { +// return groupId; +// } +// public void setGroupId(String groupId) { +// this.groupId = groupId; +// } + public int getQueueIndex() { + return queueIndex; + } + public void setQueueIndex(int queueIndex) { + this.queueIndex = queueIndex; + } + public int getCurrentoffset() { + return currentOffset; + } + public void setCurrentoffset(int currentoffset) { + this.currentOffset = currentoffset; + } +// public int getCacheOffset() { +// return cacheOffset; +// } +// public void setCacheOffset(int cacheOffset) { +// this.cacheOffset = cacheOffset; +// } + public int getMaxOffset() { + return maxOffset; + } + public void setMaxOffset(int maxOffset) { + this.maxOffset = maxOffset; + } +// public String getTopic() { +// return topic; +// } +// public void setTopic(String topic) { +// this.topic = topic; +// } +// public TopicAndFilter getTopicAndFilter() { +// return topicAndFilter; +// } +// public void setTopicAndFilter(TopicAndFilter topicAndFilter) { +// this.topicAndFilter = topicAndFilter; +// } +// @Override +// public String toString() { +// return "Offset: [topicAndFilter=" + topicAndFilter + ", queueIndex=" + queueIndex +// + ", currentoffset=" + currentOffset + ", MaxOffset=" + maxOffset +// + "]"; +// } + @Override + public String toString() { + return "Offset: [topic=" + topic+"filter:"+filter + ", queueIndex=" + queueIndex + + ", currentoffset=" + currentOffset + ", MaxOffset=" + maxOffset + + "]"; + } +// @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + maxOffset; + result = prime * result + currentOffset; +// result = prime * result + ((queueIndex == 0) ? 0 : queueIndex.hashCode()); +// result = prime * result + ((topicAndFilter == null) ? 0 : topicAndFilter.hashCode()); + result = prime * result + ((topic == null) ? 0 : topic.hashCode()); + result = prime * result + ((filter == null) ? 0 : filter.hashCode()); + return result; + } + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Offset other = (Offset) obj; + if (maxOffset != other.maxOffset) + return false; +// if (cacheOffset != other.cacheOffset) +// return false; + if (currentOffset != other.currentOffset) + return false; +// if (topicAndFilter == null) { +// if (other.topicAndFilter != null) +// return false; +// } else if (!topicAndFilter.equals(other.topicAndFilter)) +// return false; + if (queueIndex != other.queueIndex) { + return false; + } + if (topic == null) { + if (other.topic != null) + return false; + } else if (!topic.equals(other.topic)) + return false; + return true; + } + + + + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/broker/function/ReSendHandler.java b/src/main/java/com/alibaba/middleware/race/mom/broker/function/ReSendHandler.java new file mode 100644 index 0000000..f570020 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/broker/function/ReSendHandler.java @@ -0,0 +1,88 @@ +//package com.alibaba.middleware.race.mom.broker.function; +// +//import io.netty.channel.Channel; +// +//import java.util.concurrent.BlockingQueue; +//import java.util.concurrent.ExecutorService; +//import java.util.concurrent.Executors; +//import java.util.concurrent.LinkedBlockingQueue; +// +//import com.alibaba.middleware.race.mom.Message; +//import com.alibaba.middleware.race.mom.broker.group.ConsumerGroup; +//import com.alibaba.middleware.race.mom.util.TopicAndFilter; +///** +// * 每个集群有自己的ReSendHandler +// * @author youngforever +// * +// */ +//public class ReSendHandler { +// private TopicAndFilter topicAndFilter; +// private ConsumerGroup group; +// public BlockingQueue reSendQueue=new LinkedBlockingQueue(); +// +// int reSendThreadPoolSize=2; +// private ExecutorService reSendPool = Executors.newFixedThreadPool(reSendThreadPoolSize); +// +// +// public ReSendHandler(TopicAndFilter topicAndFilter, ConsumerGroup group) { +// this.topicAndFilter = topicAndFilter; +// this.group = group; +// } +// public void consumerLostReSend(){ +// +// } +// public void timeoutReSend(){ +// +// } +// public void failReSend(){ +// +// } +// +// +// /** +// * 开始发送任务 +// */ +// public void startRend(){ +// for(int i=0;i recover(){ + return recoverMessage(makeOffsets(recoveroffsets())); + } + + public ArrayList recoveroffsets(){ + ArrayList offsets=substore.read(); + System.out.println(offsets); + return offsets; + } + /** + * 集群组 进度重叠交叉等,对其取并集,避免从磁盘重复读取消息 + * 遍历所有恢复的Offset,做其它恢复的准备和恢复消息去重 + * @param offsets + * @return + */ + public List makeOffsets(ArrayListoffsets ){ + HashMapmakedOffsets=new HashMap<>(); + HashMap>producerGroupRecoverInfos=new HashMap<>(); + Offset makedoffset; + TopicAndFilter topicAndFilter; + for(Offset offset:offsets){ + topicAndFilter=new TopicAndFilter(offset.getTopic(),offset.getFilter()); + + System.out.println(offset); + /* + * 恢复存储进度关系 + */ + if(producerGroupRecoverInfos.containsKey(topicAndFilter)){ + producerGroupRecoverInfos.get(topicAndFilter).put(offset.getQueueIndex(),offset.getMaxOffset()); + }else{ + Map maxoffsets=new HashMap<>(); + maxoffsets.put(offset.getQueueIndex(), offset.getMaxOffset()); + producerGroupRecoverInfos.put(topicAndFilter, maxoffsets); + } + + if(makedOffsets.containsKey(topicAndFilter)){ + makedoffset=makedOffsets.get(topicAndFilter); + if(offset.getCurrentoffset()makedoffset.getMaxOffset()){ + makedoffset.setMaxOffset(offset.getMaxOffset()); + } + }else { + makedOffsets.put(topicAndFilter,offset); + } + } + ProducerGroup.recover(producerGroupRecoverInfos); + + return Arrays.asList(makedOffsets.values().toArray(new Offset[0])); + } + + public ConcurrentHashMap recoverMessage(List makedOffsets){ + long startTime = System.currentTimeMillis(); + ConcurrentHashMap msgMap=new ConcurrentHashMap<>(1024*128); + System.out.println("offsets:"+makedOffsets); + int recoverNum=0; + int queueId; + TopicAndFilter topicAndFilter; + for(Offset offset:makedOffsets){ + topicAndFilter=new TopicAndFilter(offset.getTopic(),offset.getFilter()); + queueId=offset.getQueueIndex(); +// //恢复topicAndgroup2subScribe +// LinkedList offsetList; +// if(!topicAndgroup2subScribe.containsKey(topic+"@"+group)){ +// offsetList=new LinkedList(); +// topicAndgroup2subScribe.put(topic+"@"+group, offsetList); +// }else{ +// offsetList=topicAndgroup2subScribe.get(topic+"@"+group); +// } +// offsetList.add(Integer.parseInt(queueId),offset); +// //恢复topicAndgroup2sendMsgQueue +// LinkedList> sendQueuesList; +// if(!topicAndgroup2sendMsgQueue.containsKey(topic + "@" + group)){ +// sendQueuesList=new LinkedList>(); +// topicAndgroup2sendMsgQueue.put(topic + "@" + group, sendQueuesList); +// }else +// { +// sendQueuesList=topicAndgroup2sendMsgQueue.get(topic + "@" + group); +// } +// BlockingQueuesendQueue=new LinkedBlockingQueue(); +// sendQueuesList.add(Integer.parseInt(queueId),sendQueue); + //消息入内存 + int currentOffset=offset.getCurrentoffset(); + int MaxOffset=offset.getMaxOffset(); + Message msg; + int i=0;//currentOffset偏移量 + List blist = mstore.readByteNormal(topicAndFilter.toString(),String.valueOf(queueId),currentOffset+1, MaxOffset); + recoverNum+=blist.size(); + System.out.println(blist.size()); + for(byte[] b:blist){ + try{ + msg=(Message)fst.asObject(b); + System.out.println("恢复消息"+msg); + }catch(Exception e){ + continue; + } + msgMap.put(new TopicAndFilter(msg.getTopic(),msg.getProperties()).toString(), msg); +// msgid2offset.put(msg.getMsgId(), currentOffset+i); + i++; + } + + } + long endTime = System.currentTimeMillis(); + System.out.println("恢复cost：" + (endTime - startTime)+" ,一共恢复"+recoverNum+"条"); + + return msgMap; + } + public int getMaxOffset() { + return maxOffset; + } + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/broker/function/SendInfo.java b/src/main/java/com/alibaba/middleware/race/mom/broker/function/SendInfo.java new file mode 100644 index 0000000..40f2ff1 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/broker/function/SendInfo.java @@ -0,0 +1,136 @@ +package com.alibaba.middleware.race.mom.broker.function; + +import java.io.Serializable; +import java.util.concurrent.atomic.AtomicInteger; + +import com.alibaba.middleware.race.mom.SendStatus; +import com.alibaba.middleware.race.mom.config.Config; +import com.alibaba.middleware.race.mom.util.TopicAndFilter; + +public class SendInfo implements Serializable { + /** + * 发送结果，判断消费是否超时等 + */ + private static final long serialVersionUID = 321275686710864167L; + +// public static final Map sendInfoMap = new ConcurrentHashMap<>(); + private MessageInfo msgInfo; + private String info; + private SendStatus status=SendStatus.SEND; +// private int msgOffset; +// private String topic; + /* + * 订阅集群的topicAndFilter + */ + private TopicAndFilter topicAndFilter; +// private int queueId; + + private long sendTime; +// private long recvAckTime; + private long timeoutLimit=Config.timeoutLimit; + private AtomicInteger reSendCounter=new AtomicInteger(); + + + public SendInfo(MessageInfo msgInfo){ + this.msgInfo=msgInfo; + } + /** + * @param timeoutLimit unit:ms 允许的超时时间限制 + */ + public SendInfo(MessageInfo msgInfo,long timeoutLimit){ + this.msgInfo=msgInfo; + this.timeoutLimit=timeoutLimit; + } + + /** + * 重置snedResult,用于重投时重投次数增加、重置发送时间，状态等 + * + */ + public void reSend(){ + sendTime=System.currentTimeMillis(); +// recvAckTime=0; + reSendCounter.incrementAndGet(); + this.status=SendStatus.RESEND; + } + public String getInfo() { + return info; + } + public SendStatus getStatus() { + return status; + } + public void setInfo(String info) { + this.info = info; + } + public void setStatus(SendStatus status) { + this.status= status ; + } + +// public long getRecvAckTime() { +// return recvAckTime; +// } +// public void setRecvAckTime(long recvAckTime) { +// this.recvAckTime = recvAckTime; +// } + public void setSendTime(long sendTime){ + this.sendTime=sendTime; + } + public long getSendTime(){ + return this.sendTime; + } + public int getMsgOffset() { + return this.msgInfo.getOffset(); + } + public TopicAndFilter getTopicAndFilter() { + return topicAndFilter; + } + public void setTopicAndFilter(TopicAndFilter topicAndFilter) { + this.topicAndFilter = topicAndFilter; + } + public int getQueueId() { + return this.msgInfo.getQueueIndex(); + } + + public MessageInfo getMsgInfo() { + return msgInfo; + } + public int getReSendCount() { + return reSendCounter.get(); + } +// public void decreaseNeedSendGroupsNum(){ +// needSendGroupsNum.decrementAndGet(); +// } + /** + * @param nowTime + * @return 超时则 返回true;否则返回false; + */ + public boolean isTimeout(){ + if(System.currentTimeMillis()-sendTime>=this.timeoutLimit){ + this.status=SendStatus.TIMEOUT; + return true; + }else return false; + } +// public boolean canRemove(){ +// if(status==SendStatus.SUCCESS){ +// return true; +// }else if(this.reSendCounter.get()>=Config.maxReSendTime){ +// return true; +// }else return false; +// } + /** + * @return 超时则 返回true;否则返回false; + * @throws Exception :throw exception if do not receive consumer's ACK or do not set the receive ACK time:recvAckTime + */ +// public boolean isTimeout() throws Exception{ +// if(this.recvAckTime!=0){ +// return recvAckTime-sendTime>=this.timeoutLimit; +// }else { +// throw new Exception("do not recieve consumer's ACK or do not set the recieve ACK time:recvAckTime "); +// } +// } +// + @Override + public String toString(){ + return "msg "+msgInfo.getMsg()+" send "+status+" info:"+info; + } + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/broker/function/SendInfoScanner.java b/src/main/java/com/alibaba/middleware/race/mom/broker/function/SendInfoScanner.java new file mode 100644 index 0000000..0e9d4fa --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/broker/function/SendInfoScanner.java @@ -0,0 +1,191 @@ +package com.alibaba.middleware.race.mom.broker.function; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicInteger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.middleware.race.mom.SendStatus; +import com.alibaba.middleware.race.mom.broker.group.ConsumerGroup; +import com.alibaba.middleware.race.mom.config.Config; +import com.alibaba.middleware.race.mom.store.ArrayStore; +import com.alibaba.middleware.race.mom.store.ArrayStoreImp; +import com.alibaba.middleware.race.mom.store.OffsetNum; + + +/** + * 发送状态扫描器 + * 一.负责检测消费超时,失败等,如果超时或失败加入重投 + * 二.负责内存检测,一旦内存堆积过多超过内存大小警戒线,超时堆积消息将移除,保证系统正常运行以及正常消费tps + * @author young + * + */ +public class SendInfoScanner { + private static Logger logger = LoggerFactory.getLogger(SendInfoScanner.class); + + private ConsumerGroup group; + private Timer timoutScannerTimer; + /** + * 重投配置 + */ + private int timeoutLimit=10000; + private int maxReSendTime=10000000; + /** + * 重发线程池大小,也许可以调整大小来削峰谷 + */ + private int reSendPoolSize=2; + private ExecutorService reSendPool; + private boolean reSendStarted=false; + /** + * 重投队列,死信队列,记得刷盘...TO DO....................................................................... + */ + private BlockingQueue reSendQueue=new LinkedBlockingQueue(); + private BlockingQueue deadMessageQueue=new LinkedBlockingQueue(); + + private ArrayStore arrayStore=new ArrayStoreImp(); + private AtomicInteger reSendFailCount=new AtomicInteger(); + + /** + * 重投恢复 + */ + ExecutorService recoverPool=Executors.newFixedThreadPool(1); + public SendInfoScanner(ConsumerGroup group){ + this.group=group; + } + public void start(){ + scanSendInfoMap(); + } + public void scanSendInfoMap(){ + if(timoutScannerTimer==null){ + timoutScannerTimer=new Timer(); + } + timoutScannerTimer.schedule(new TimerTask(){ + final Map >> topicAndFilter2queues=new HashMap<>(); + Map> queue2offsets; + ArrayList offsets; + Map sendInfoMap=group.getSendInfoMap(); + @Override + public void run() { + SendInfo sendInfo; + for(Map.Entry entry:sendInfoMap.entrySet()){ + sendInfo=entry.getValue(); + if(sendInfo.getStatus()==SendStatus.TIMEOUT||sendInfo.getStatus()==SendStatus.FAIL||sendInfo.isTimeout()){ + if(sendInfo.getReSendCount()5000){ +// logger.error("内存超出"+sendInfoMap.size()); +// /* +// * 内存不足,存储重投部分offset +// */ +// if((queue2offsets=topicAndFilter2queues.get(sendInfo.getTopicAndFilter().toString()))!=null){ +// if((offsets=queue2offsets.get(String.valueOf(sendInfo.getQueueId())))!=null){ +// offsets.add(new OffsetNum().setI(sendInfo.getMsgOffset())); +// }else{ +// offsets=new ArrayList(); +// offsets.add(new OffsetNum().setI(sendInfo.getMsgOffset())); +// queue2offsets.put(String.valueOf(sendInfo.getQueueId()), offsets); +// } +// }else { +// offsets=new ArrayList(); +// offsets.add(new OffsetNum().setI(sendInfo.getMsgOffset())); +// queue2offsets=new HashMap<>(); +// queue2offsets.put(String.valueOf(sendInfo.getQueueId()), offsets); +// topicAndFilter2queues.put(sendInfo.getTopicAndFilter().toString(), queue2offsets); +// } +// sendInfoMap.remove(entry.getKey()); +// /* +// * 死信队列暂时先简单清空.................................... +// */ +// deadMessageQueue.clear(); +// }else + { + reSendQueue.add(sendInfo.getMsgInfo());//加入重投队列 +// logger.error(sendInfo.getMsgInfo().getMsg()+"加入重投队列"); + recoverPool.execute(new Runnable(){ + + @Override + public void run() { + ArrayList offsets=null; + // TODO Auto-generated method stub + for(int i=0;i>> queue2offsets:topicAndFilter2queues.entrySet()){ + for(Map.Entry> offsets:queue2offsets.getValue().entrySet()){ + arrayStore.store(offsets.getValue(), queue2offsets.getKey(), offsets.getKey()); + } + } + + + } + },timeoutLimit+100,1000); + } + + /** + * 重投 + * @param reSendQueue + * @param consumers + * @param offset + */ + public void startReSend(){ + //第一次开启重投时创建线程池 + reSendPool = Executors.newFixedThreadPool(reSendPoolSize/*Runtime.getRuntime().availableProcessors()+2*/); + for(int i=0;i waitAckNumMap=new HashMap<>(); + private static int i=0; + private Map> waitAckMap=new HashMap<>(); + private Map waitAcksMap=new HashMap<>(); + private Map/*sendId or ack String*/> waitAckGroupMap=new HashMap<>(); +// private StringBuilder acks=new StringBuilder(); + + + /** + * 一次存储成功回调,存储完成(强行force)后异步返回对应的ack + */ + public void onStoreSucceed(TopicAndFilter topicAndFilter,int queueIndex,int offset){ + /** + * ack返回 + */ + sendACKPool.execute(new Runnable(){ + @Override + public void run() { +// for(Channel producer:waitAckMap.keySet()){ +// for(String msgId:waitAckMap.get(producer)){ +// producer.writeAndFlush(msgId + "\r\n"); +// i++; +//// System.out.println("存储成功,返回ack"+i+"条"); +// } +// } +// for(Map.Entry acks:waitAcksMap.entrySet()){ +// acks.getKey().writeAndFlush(acks.getValue().toString()+"\r\n"); +// logger.error("sendback acks"+acks.getValue().toString()); +// } +// for(Map.Entryacks:waitAckNumMap.entrySet()){ +// acks.getKey().writeAndFlush(acks.getValue()+"\r\n"); +//// logger.error("sendback acks"+acks.getValue().toString()); +// } + for(Map.Entry >ack:waitAckGroupMap.entrySet()){ + StringBuilder acks=new StringBuilder(); + for(String id:ack.getValue()) + { + acks.append("@"); + acks.append(id); + } + ack.getKey().writeAndFlush(acks+"\r\n"); + //System.out.println("返回的是:"+acks); +// logger.error("sendback acks"+acks.getValue().toString()); + } + } + }); + /** + * 精确发送已存储的消息 + */ + ConcurrentHashMap needSendGroups=ConsumerGroup.getNeedSendGroups(topicAndFilter); + if(needSendGroups!=null) + for(ConsumerGroup consumerGroup:needSendGroups.values()){ +// consumerGroup.startSend(); + consumerGroup.sendSpecificMsgs(queueIndex, offset); + /** + * 存储到文件,再来设置MaxOffset + */ + consumerGroup.setMaxOffset( queueIndex,offset); + } + } + /** + * 添加ack等待 + */ + public void addAckWaiter(Channel producer,String msgId){ + List msgIds=waitAckMap.get(producer); + if(msgIds==null){ + msgIds=new ArrayList<>(); + msgIds.add(msgId); + waitAckMap.put(producer,msgIds); + }else { + msgIds.add(msgId); + } + } + public void addAckWaiterGroup(Channel producer,String msgId){ + StringBuilder acks=waitAcksMap.get(producer); + if(acks==null){ + acks=new StringBuilder(); + acks.append(msgId).append(" "); + waitAcksMap.put(producer, acks); + }else acks.append(msgId).append(" "); + } + public void addAckWaiterNum(Channel producer){ + Integer ackNum=waitAckNumMap.get(producer); + logger.debug(String.valueOf(ackNum)); + if(ackNum==null){ + ackNum=new Integer(1); + waitAckNumMap.put(producer, ackNum); + }else waitAckNumMap.put(producer, ++ackNum); + } + public void addAckGroupWaiter(Channel producer,ArrayList sendId){ + List ack=waitAckGroupMap.get(producer); + logger.debug(String.valueOf(ack)); + if(ack==null){ + waitAckGroupMap.put(producer, sendId); + }else if(!ack.equals(sendId)){ + logger.error(" 两次存储并未串行，可能需要解决异步调用"); + } + } +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/broker/function/SubsStore.java b/src/main/java/com/alibaba/middleware/race/mom/broker/function/SubsStore.java new file mode 100644 index 0000000..6547a4c --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/broker/function/SubsStore.java @@ -0,0 +1,44 @@ +package com.alibaba.middleware.race.mom.broker.function; + +import java.util.ArrayList; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.middleware.race.mom.broker.group.ConsumerGroup; +import com.alibaba.middleware.race.mom.store.SubscribeStore; +import com.alibaba.middleware.race.mom.store.SubscribeStoreImp; + +public class SubsStore { + private static Logger logger = LoggerFactory.getLogger(SubsStore.class); + private static SubscribeStore substore=new SubscribeStoreImp(); + private static Timer timer=new Timer(); + /** + * 定时任务-存储订阅关系 + * @param start + * @param during + */ + public static void subscribeStoreStart(){ + + + timer.schedule(new TimerTask(){ + @Override + public void run() { + logger.error("异步-订阅关系/消费进度定时持久化任务"); + ArrayList sublist=new ArrayList<>(); + for(ConcurrentHashMap consumerGroups:ConsumerGroup.topicAndFilter2groups.values()){ + for (ConsumerGroup consumerGroup :consumerGroups.values()){ + //System.out.println("key= " + entry.getKey() + " and value= " + entry.getValue()); + sublist.addAll(consumerGroup.getConsumeOffsets()); + } + } + + substore.write(sublist); + logger.error("订阅消费进度关系刷盘"+sublist); + } + },1000, 1000); + } +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/broker/group/ConsumerGroup.java b/src/main/java/com/alibaba/middleware/race/mom/broker/group/ConsumerGroup.java new file mode 100644 index 0000000..659e650 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/broker/group/ConsumerGroup.java @@ -0,0 +1,534 @@ +package com.alibaba.middleware.race.mom.broker.group; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; + +import java.util.LinkedList; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.middleware.race.mom.ConsumeResult; +import com.alibaba.middleware.race.mom.Message; +import com.alibaba.middleware.race.mom.SendStatus; +import com.alibaba.middleware.race.mom.broker.function.MessageInfo; +import com.alibaba.middleware.race.mom.broker.function.Offset; +import com.alibaba.middleware.race.mom.broker.function.SendInfo; +import com.alibaba.middleware.race.mom.broker.function.SendInfoScanner; +import com.alibaba.middleware.race.mom.config.Config; +import com.alibaba.middleware.race.mom.util.TopicAndFilter; +/** + * 消费者集群管理 + * @author young + * + */ +/** + * 消费者集群管理 + * @author young + * + */ +public class ConsumerGroup { + private static Logger logger = LoggerFactory.getLogger(ConsumerGroup.class); + + /** + * 所有用于发送的 线程池可分配大小,也可以不限制大小,但也许应给避免订阅集群过多,发送线程池过大,不断优化测试..... + */ + private static int sendThreadPoolTotalSize=10; + + + /** + * 每个订阅集群 + */ + private LinkedListconsumers = new LinkedList(); + private String groupId; + private String subscribedTopic; + private Map subscribedFilter; + private TopicAndFilter topicAndFilter; + /** + * 每个集群的发送信息(发送状态,失败,成功,超时等) + */ + private Map sendInfoMap = new ConcurrentHashMap<>(); + /** + * 重投 + */ + private SendInfoScanner sendInfoScanner=new SendInfoScanner(this); + +// private Map> topicAndgroup2subScribe = new ConcurrentHashMap<>(); + /* + * 每个集群的sendQueueNum个发送队列,对应消息拆分存储为sendQueueNum份文件 + * sendThreadPoolSize为分配改当前集群发送线程池大小,不断优化测试..... + * sendPool,负责当前集群发送的线程池 + */ + /** + * 发送队列个数,与存储队列一一对应,须全局统一配置(错了将报错,注意!!!!) + */ + private int sendQueueNum=Config.maxqueue; +// private int sendThreadPoolSize=sendQueueNum*4; + private int sendThreadPoolSize=Runtime.getRuntime().availableProcessors()/2; + private LinkedList> sendQueues = new LinkedList<>(); + /** + * 当前集群(组)的消费进度情况 + */ + private LinkedList consumeOffsets=new LinkedList (); + + private int sendQueueBalancer=-1; + private ExecutorService sendPool = Executors.newFixedThreadPool(sendThreadPoolSize); + + + + /** + * 构造函数 + * @param subscribedTopic + * @param subscribedFilter + * @param groupId + */ + public ConsumerGroup(String subscribedTopic,Map subscribedFilter,String groupId){ + this.subscribedTopic=subscribedTopic; + this.subscribedFilter=subscribedFilter; + topicAndFilter=new TopicAndFilter(subscribedTopic,subscribedFilter); + this.groupId=groupId; + /* + * 集群创建时初始化队列,初始化对应的offet进度 + */ + Offset offset; + for(int i=0;i()); +// offset=new Offset(subscribedTopic,groupId,topicAndFilter.toString()); + offset=new Offset(); + offset.setQueueIndex(i); +// offset.setTopic(subscribedTopic); + this.consumeOffsets.add(offset); + } + /* + * 启动发送任务,不采用立即转发方式,旺旺群讨论的结果 + */ +// startSend(); + /* + * 启动重投检测 + */ + +// sendInfoScanner.start(); + } + /** + * 负载均衡,且去除死掉的消费者 + * @return 本次发往的消费者.如果集群都不在线,则返回null + */ + private volatile int consumerBalancer=-1; + public synchronized Channel consumersBalance(){ + consumerBalancer++; + if(consumerBalancer==consumers.size()){ + consumerBalancer=0; + } + Channel consumer=consumers.get(consumerBalancer); + if (consumer==null) return null; + if(!consumer.isOpen()){ + logger.error("消费者失去连接: " + consumer.hashCode()); + consumers.remove(consumer); + return consumersBalance(); + }else return consumer; + + } + /** + * 发送队列均衡 + * @return + */ + public synchronized BlockingQueue sendQueuesBalance(){ + sendQueueBalancer++; + if(sendQueueBalancer==sendQueueNum){ + sendQueueBalancer=0; + } + BlockingQueue sendQueue=sendQueues.get(sendQueueBalancer); + return sendQueue; + } + /** + * 开始发送任务,此方式为收到消息,立即转发,与存储并行 + */ + public void startSend(){ + for(final BlockingQueue sendQueue:sendQueues){ + for(int i=0;i getConsumeOffsets() { + return consumeOffsets; + } + public String getSubscribedTopic() { + return subscribedTopic; + } + public void setSubscribedTopic(String subscribedTopic) { + this.subscribedTopic = subscribedTopic; + } + public Map getSubscribedFilter() { + return subscribedFilter; + } + public Map getSendInfoMap() { + return sendInfoMap; + } + public void setSendInfoMap(Map sendInfoMap) { + this.sendInfoMap = sendInfoMap; + } + public void setSubscribedFilter(Map subscribedFilter) { + this.subscribedFilter = subscribedFilter; + } + public LinkedList getConsumers() { + return consumers; + } + public void setConsumers(LinkedList consumers) { + this.consumers = consumers; + } + + public int getSendQueueNum() { + return sendQueueNum; + } + public void setSendQueueNum(int sendQueueNum) { + this.sendQueueNum = sendQueueNum; + } + public LinkedList> getSendQueues() { + return sendQueues; + } + public void setSendQueues(LinkedList> sendQueues) { + this.sendQueues = sendQueues; + } + public int getSendQueueBalancer() { + return sendQueueBalancer; + } + public TopicAndFilter getTopicAndFilter() { + return topicAndFilter; + } + public void setTopicAndFilter(TopicAndFilter topicAndFilter) { + this.topicAndFilter = topicAndFilter; + } + public void setSendQueueBalancer(int sendQueueBalancer) { + this.sendQueueBalancer = sendQueueBalancer; + } + public String toString(){ + return "subscribedTopic"+subscribedTopic+" subscribedFilter"+subscribedFilter; + } +/** + * 非静态部分 为某一具体集群 + * 静态部分 属于集群组,管理所有订阅集群 + */ + /** + * 订阅集群组,每种topicAndfilter可能有一个或多个订阅集群 + */ + public static Map> topicAndFilter2groups = new ConcurrentHashMap<>(); + /** + * 添加 topicAndFilter的订阅集群 + * @param topicAndFilter + * @param groupId + */ + public static ConsumerGroup addGroup(TopicAndFilter topicAndFilter,String groupId){ + Map groups=topicAndFilter2groups.get(topicAndFilter.toString());//集群组 + ConsumerGroup group=new ConsumerGroup(topicAndFilter.getTopic(),topicAndFilter.getFilter(),groupId); + if(groups==null){ + groups=new ConcurrentHashMap(); + groups.put(groupId,group); + topicAndFilter2groups.put(topicAndFilter.toString(), (ConcurrentHashMap) groups); + }else if(!groups.containsKey(groupId)){ + groups.put(groupId, group); + } +// else {//集群已经存在,这种情况可能发生在集群全部掉线,而后恢复重连.或者集群新增消费者 +// group=groups.get(groupId); +// } + return group; + } + public static ConcurrentHashMap getGroups(TopicAndFilter topicAndFilter){ + return topicAndFilter2groups.get(topicAndFilter.toString()); + } + public static ConsumerGroup getGroup(TopicAndFilter topicAndFilter,String groupId){ + Map groups=getGroups(topicAndFilter); + Map groupsNoFilter=getGroups(new TopicAndFilter(topicAndFilter.getTopic(),null)); + ConsumerGroup group; + if(groupsNoFilter==null&&groups==null){ + return null; + }else if(groups!=null&&(group=groups.get(groupId))!=null){ + return group; + }else if(groupsNoFilter!=null){ + return groupsNoFilter.get(groupId); + }else return null; + } + /** + * 取得filter 和null两种needSend订阅集群组 + * @param topicAndFilter + * @return + */ + public static ConcurrentHashMap getNeedSendGroups(TopicAndFilter topicAndFilter){ + ConcurrentHashMap groups=getGroups(topicAndFilter); + TopicAndFilter topicNoFilter=new TopicAndFilter (topicAndFilter.getTopic(),null); + ConcurrentHashMap noFilterGroups=getGroups(topicNoFilter); + + + if(groups!=null&&noFilterGroups!=null){ + groups.putAll(getGroups(topicNoFilter)); + }else if(noFilterGroups!=null){ + groups=noFilterGroups; + } +// if(groups!=null) +// for(ConsumerGroup group:groups.values()){ +// if(group.consumers.isEmpty()){ +// groups.remove(group.getGroupId()); +// } +// } + return groups; + } + /** + * 取得 此topicAndFilter下所有负载均衡后须发送的消费者 + * @param topicAndFilter + * @return + */ + public static LinkedList getNeedSendBalancedConsumers(TopicAndFilter topicAndFilter){ + LinkedList needSendBalancedConsumers=new LinkedList(); + Mapgroups=getNeedSendGroups(topicAndFilter); + for(ConsumerGroup consumerGroup:groups.values()){//每个须发送集群都要负载均衡找出消费者 + Channel consumer=consumerGroup.consumersBalance(); + if(consumer!=null){ + needSendBalancedConsumers.add(consumer); + }else { + logger.error("消费者集群"+consumerGroup+"全部失去连接"); + } + } + return needSendBalancedConsumers; + } + public static int needSendGroupsNum(TopicAndFilter topicAndFilter){ + return getNeedSendGroups(topicAndFilter).size(); + } +// public static boolean contains(TopicAndFilter topicAndFilter){ +// return topicAndFilter2groups.containsKey(topicAndFilter.toString()); +// } + /** + * 注册订阅者(消费者) + * @param consumer + * @param groupId + * @param topicAndFilter + */ + public static void addConsumer(Channel consumer ,String groupId,TopicAndFilter topicAndFilter){ + ConsumerGroup group=getGroup(topicAndFilter,groupId); + if(group!=null){ + if(group.consumers.size()==0){ + logger.error("消费者集群全部down机后恢复,触发消费者离线重投"); + /** + * TO DO.......................................................... + */ + } + group.addConsumer(consumer); + }else addGroup(topicAndFilter,groupId).addConsumer(consumer); + } + /** + * 消费结果总处理 + * @param consumeResult + * @param ctx + */ + public static void confirmConsumer(ConsumeResult consumeResult, ChannelHandlerContext ctx){ + logger.debug("recv consume result"); + String msgId = consumeResult.getMsgId(); + ConsumerGroup group=ConsumerGroup.getGroup(consumeResult.getTopicAndFilter(),consumeResult.getGroupId()); + SendInfo sendInfo=group.sendInfoMap.get(msgId); + if(sendInfo==null) return; + switch (consumeResult.getStatus()){ + case SUCCESS:sendInfo.setStatus( SendStatus.SUCCESS); break; + case FAIL:sendInfo.setStatus( SendStatus.FAIL); break; + default:logger.error("消费者 无消费状态.......");break; + } + group.handleConsumeResult(sendInfo, ctx); + } + /** + * 准备消息发送(进入发送队列),存储完成后发送 + * @param topicAndFilter + * @param msgInfo + */ + private static long waitSubStartTime; + public static void sendMsg(TopicAndFilter topicAndFilter,MessageInfo msgInfo){ + //获取消息应给发往的订阅集群组,即所有订阅此消息的集群 + Map groups=ConsumerGroup.getNeedSendGroups(topicAndFilter); + if(groups==null){ + if(waitSubStartTime==0){ + waitSubStartTime=System.currentTimeMillis(); + }else if(System.currentTimeMillis()-waitSubStartTime>Config.noConsumerWaitTimeLimit){//800ms后仍无订阅者信息,则递归结束, + msgInfo.getProducer().writeAndFlush("fail@"+msgInfo.getMsg().getMsgId()+ "\r\n"); + logger.error(Config.noConsumerWaitTimeLimit+"ms 任无此"+topicAndFilter+"的订阅集群(组)发起订阅"); + return; + } + try { + Thread.sleep(1); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + //logger.error("没有此"+topicAndFilter+"的订阅集群(组)"); + sendMsg(topicAndFilter,msgInfo);//递归不断尝试获取订阅者集群 + }else{ + //设置需要发送的订阅集群数 +// msgInfo.setSubGroupsCount(groups.size()); + //加入各订阅集群的发送队列,自动发送给消费者 + logger.debug((System.currentTimeMillis()-waitSubStartTime)+"发现订阅信息"); + for(ConsumerGroup consumerGroup:groups.values()){ + consumerGroup.addToSendQueue(topicAndFilter,msgInfo); + } + } + waitSubStartTime=0; + } + +// /** +// * 以groupId区分俩个组 +// */ +// @Override +// public boolean equals(Object obj) { +// if(obj instanceof ConsumerGroup){ +// if(((ConsumerGroup) obj).getGroupId().equals(this.groupId))return true; +// else return false; +// }else return false; +// } + + + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/broker/group/ProducerGroup.java b/src/main/java/com/alibaba/middleware/race/mom/broker/group/ProducerGroup.java new file mode 100644 index 0000000..9926d3a --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/broker/group/ProducerGroup.java @@ -0,0 +1,226 @@ +package com.alibaba.middleware.race.mom.broker.group; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.middleware.race.mom.broker.function.MessageInfo; +import com.alibaba.middleware.race.mom.broker.function.MessageInfoQueue; +import com.alibaba.middleware.race.mom.broker.function.MessageSend; +import com.alibaba.middleware.race.mom.broker.function.MessageStore; +import com.alibaba.middleware.race.mom.util.TopicAndFilter; + + +/** + * 生产者集群(组)管理 + * 假设存在多个集群生产同一种topicAndfilter的消息,broker端可将集群组视为同一集群,不做集群区分,只以topicAndfilter区分 + * 消息存储以topicAndfilter将生产者消息归类存储 + * ack返回,我们假定必须返回相应生产者(消息的直接生产者),而不是集群中任一生产者 + * + * @author youngforever + */ +public class ProducerGroup { + private static Logger logger = LoggerFactory.getLogger(ProducerGroup.class); + /** + * 每个生产者集群 + */ +// private Mapproducers = new HashMap<>(); +// private String groupId; +// private String produceTopic; +// private Map produceFilter; + private TopicAndFilter produceTopicAndFilter; + private MessageStore msgStore ; + + /** + * 私有构造函数,仅用于恢复时 + * @param produceTopic + * @param produceFilter + * @param queueMaxOffsets + */ + private ProducerGroup(TopicAndFilter topicAndFilter,Map queueMaxOffsets){ + this.produceTopicAndFilter=topicAndFilter; + this.msgStore=new MessageStore(produceTopicAndFilter, queueMaxOffsets); + + } + /** + * 用于新增生产者集群 + * @param produceTopic + * @param produceFilter + * @param queueMaxOffsets + */ + public ProducerGroup(String produceTopic,Map produceFilter){ +// this.produceTopic=produceTopic; +// this.produceFilter=produceFilter; + this.produceTopicAndFilter=new TopicAndFilter(produceTopic,produceFilter); + this.msgStore=new MessageStore(produceTopicAndFilter); + } +// public void addProducer(Channel producer,String producerId){ +// if(!producers.containsKey(producerId)){ +// producers.put(producerId,producer); +// }else if(!producers.get(producerId).isOpen()){//已经存在此id 的producer,但是关闭了连接. 现在应该时发起了重连 +// producers.put(producerId,producer); +// } +// } + public String toString(){ + return produceTopicAndFilter.toString(); + } + /** + * 非静态部分 为某一具体集群 + * 静态部分 属于集群组,管理所有生产者集群 + */ + public static Map topicAndFilter2group = new ConcurrentHashMap<>(); + /** + * 添加 topicAndFilter的订阅集群 + * @param topicAndFilter + * @param groupId + */ + public static ProducerGroup addGroup(TopicAndFilter topicAndFilter){ + ProducerGroup group=topicAndFilter2group.get(topicAndFilter.toString());//集群组 + if(group==null){ + group=new ProducerGroup(topicAndFilter.getTopic(),topicAndFilter.getFilter()); + topicAndFilter2group.put(topicAndFilter.toString(),group); + } + return group; + } + /** + * 仅用于恢复时 + * @param topicAndFilter + * @param queueMaxOffsets + * @return + */ + private static ProducerGroup addGroup(TopicAndFilter topicAndFilter,Map queueMaxOffsets){ + ProducerGroup group=topicAndFilter2group.get(topicAndFilter.toString());//集群组 + if(group==null){ + group=new ProducerGroup(topicAndFilter,queueMaxOffsets); + topicAndFilter2group.put(topicAndFilter.toString(),group); + } + return group; + } + public static ProducerGroup getGroup(TopicAndFilter topicAndFilter){ + return topicAndFilter2group.get(topicAndFilter.toString()); + } + /** + * 注册生产者 + * @param producer + * @param groupId + * @param topicAndFilter + */ + /* + * 采用了 存储成功监听器机制,那么生产者实际上不用保存 + */ +// public static void addProducer(Channel producer ,String producerId,String producerGroupId,TopicAndFilter topicAndFilter){ +// ProducerGroup group=getGroup(topicAndFilter); +// if(group!=null){ +// group.addProducer(producer,producerId); +// }else addGroup(topicAndFilter).addProducer(producer,producerId); +//// System.out.println(topicAndFilter2groups); +// } + /** + * + * @param topicAndFilter + * @param msgInfo + * @return offset,消息进入持久化队列后,返回下标做为offset(代表存储顺序,对应文件消息索引,同时用于表示消费进度) + */ +// public static String storeMsg(TopicAndFilter topicAndFilter,MessageInfo msgInfo){ +// ProducerGroup producerGroup = getGroup(topicAndFilter); +// if(producerGroup==null){ +// logger.error("生产者集群(组)尚未注册"); +// producerGroup=addGroup(topicAndFilter); +// } +// return producerGroup.msgStore.storeMsg(msgInfo); +// } + /** + * 组存处理 + * @param topicAndFilter + * @param msgInfo + * @return offset（返回为maxOffset）,消息进入持久化队列后,返回下标做为offset(代表存储顺序,对应文件消息索引,同时用于表示消费进度) + */ + public static String storeMsg(TopicAndFilter topicAndFilter,MessageSend msgSend){ + ProducerGroup producerGroup = getGroup(topicAndFilter); + if(producerGroup==null){ + logger.error("生产者集群(组)尚未注册"); + producerGroup=addGroup(topicAndFilter); + } + return producerGroup.msgStore.storeMsg(msgSend); + } + /** + * 恢复 + */ + public static void recover(HashMap>producerGroupRecoverInfos){ + for(Map.Entry>queueMaxOffsets :producerGroupRecoverInfos.entrySet()){ + addGroup(queueMaxOffsets.getKey(),queueMaxOffsets.getValue()); + System.out.println(addGroup(queueMaxOffsets.getKey(),queueMaxOffsets.getValue())); + } + } + +/** + * 集群组.区分集群(先勿删!!!) + */ + +// /** +// * 订阅集群组,每种topicAndfilter可能有一个或多个订阅集群 +// */ +// public static Map> topicAndFilter2groups = new ConcurrentHashMap<>(); +// /** +// * 添加 topicAndFilter的订阅集群 +// * @param topicAndFilter +// * @param groupId +// */ +// public static ProducerGroup addGroup(TopicAndFilter topicAndFilter,String groupId){ +// Map groups=topicAndFilter2groups.get(topicAndFilter.toString());//集群组 +// ProducerGroup group=new ProducerGroup(topicAndFilter.getTopic(),topicAndFilter.getFilter(),groupId); +// if(groups==null){ +// groups=new ConcurrentHashMap(); +// groups.put(groupId,group); +// topicAndFilter2groups.put(topicAndFilter.toString(), (ConcurrentHashMap) groups); +// }else if(!groups.containsKey(groupId)){ +// groups.put(groupId, group); +// } +// return group; +// } +// public static ConcurrentHashMap getGroups(TopicAndFilter topicAndFilter){ +// return topicAndFilter2groups.get(topicAndFilter.toString()); +// } +// public static ProducerGroup getGroup(TopicAndFilter topicAndFilter,String groupId){ +// Map groups=getGroups(topicAndFilter); +// if(groups!=null){ +// return groups.get(groupId); +// }else return null; +// } +// /** +// * 注册生产者 +// * @param producer +// * @param groupId +// * @param topicAndFilter +// */ +// public static void addProducer(Channel producer ,String producerId,String producerGroupId,TopicAndFilter topicAndFilter){ +// ProducerGroup group=getGroup(topicAndFilter,producerGroupId); +// if(group!=null){ +// group.addProducer(producer,producerId); +// }else addGroup(topicAndFilter,producerGroupId).addProducer(producer,producerId); +// System.out.println(topicAndFilter2groups); +// } +// public static void storeMsg(TopicAndFilter topicAndFilter,MessageInfo msgInfo){ +// ProducerGroup producerGroup = getGroup(topicAndFilter,msgInfo.getProducerGroupId()); +// if(producerGroup!=null){ +// producerGroup.msgStoreHanler.storeCache. +// }else{ +// logger.error("生产者集群尚未注册"); +// } +// } +// /** +// * 以groupId区分俩个集群 +// */ +// @Override +// public boolean equals(Object obj) { +// if(obj instanceof ProducerGroup){ +// if(((ProducerGroup) obj).getGroupId().equals(this.groupId))return true; +// else return false; +// }else return false; +// } + + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/config/Config.java b/src/main/java/com/alibaba/middleware/race/mom/config/Config.java new file mode 100644 index 0000000..3e56426 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/config/Config.java @@ -0,0 +1,28 @@ +package com.alibaba.middleware.race.mom.config; + +public class Config { + //每个group对应的队列条数 + public static int maxqueue=1; + //内存每个队列只能装这么多条消息 + public static int maxMsgNum=30000/maxqueue; + //每次从磁盘拉取多少条 + public static int maxLoadNum=2000; + /** + * 重投次数限制 + * 达到配置次数仍然失败，则不再重投 + */ + public static int maxReSendTime=3; + /** + * 10s超时限制 + */ + public static int timeoutLimit=10000; + /** + * 磁盘拉因子，当内存使用达到虚拟机内存（-Xms配置）的memoryFullFactor倍时，消息不能再存入内存，从磁盘拉取 + */ + public static double memoryFullFactor=0.25; + /** + * 无消费者时重复尝试时间 ms,可能系统启动时,订阅信息稍有延迟 + */ + public static int noConsumerWaitTimeLimit=500; + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/serializer/JdkObjectSerializer.java b/src/main/java/com/alibaba/middleware/race/mom/serializer/JdkObjectSerializer.java new file mode 100644 index 0000000..5574dad --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/serializer/JdkObjectSerializer.java @@ -0,0 +1,59 @@ +package com.alibaba.middleware.race.mom.serializer; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; + + +public class JdkObjectSerializer{ + + + public T deserialize( final byte[] bytes , final Class clazz ) { + final ByteArrayInputStream in = new ByteArrayInputStream( bytes ); + try { + final ObjectInputStream ois = new ObjectInputStream( in ); + Object obj = ois.readObject(); + return clazz.cast( obj ); + } + catch ( final ClassNotFoundException e ) { + throw new IllegalStateException( e.getMessage() , e ); + } + catch ( final IOException e ) { + throw new IllegalStateException( e.getMessage() , e ); + } + finally { + try { + in.close(); + } + catch ( final IOException e ) { + throw new IllegalStateException( e.getMessage() , e ); + } + } + } + + public byte[] serialize( final T source ) { + + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + + try { + final ObjectOutputStream oos = new ObjectOutputStream( out ); + oos.writeObject( source ); + oos.flush(); + } + catch ( final IOException e ) { + throw new IllegalStateException( e.getMessage() , e ); + } + finally { + try { + out.close(); + } + catch ( final IOException e ) { + throw new IllegalStateException( e.getMessage() , e ); + } + } + + return out.toByteArray(); + } +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/serializer/KryoxSerializer.java b/src/main/java/com/alibaba/middleware/race/mom/serializer/KryoxSerializer.java new file mode 100644 index 0000000..7acd013 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/serializer/KryoxSerializer.java @@ -0,0 +1,59 @@ +package com.alibaba.middleware.race.mom.serializer; + +import java.lang.reflect.Constructor; +import java.util.concurrent.ConcurrentHashMap; + +import sun.reflect.ReflectionFactory; + +import com.esotericsoftware.kryo.Kryo; + +public class KryoxSerializer extends Kryo { + + private final ReflectionFactory REFLECTION_FACTORY = ReflectionFactory + .getReflectionFactory(); + + private final ConcurrentHashMap, Constructor> _constructors = new ConcurrentHashMap, Constructor>(); + + @Override + public T newInstance(Class type) { + try { + return super.newInstance(type); + } catch (Exception e) { + return (T) newInstanceFromReflectionFactory(type); + } + } + + private Object newInstanceFrom(Constructor constructor) { + try { + return constructor.newInstance(); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + @SuppressWarnings("unchecked") + public T newInstanceFromReflectionFactory(Class type) { + Constructor constructor = _constructors.get(type); + if (constructor == null) { + constructor = newConstructorForSerialization(type); + Constructor saved = _constructors.putIfAbsent(type, constructor); + if(saved!=null) + constructor=saved; + } + return (T) newInstanceFrom(constructor); + } + + private Constructor newConstructorForSerialization( + Class type) { + try { + Constructor constructor = REFLECTION_FACTORY + .newConstructorForSerialization(type, + Object.class.getDeclaredConstructor()); + constructor.setAccessible(true); + return constructor; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/serializer/RpcDecoder.java b/src/main/java/com/alibaba/middleware/race/mom/serializer/RpcDecoder.java new file mode 100644 index 0000000..2abe12d --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/serializer/RpcDecoder.java @@ -0,0 +1,48 @@ +package com.alibaba.middleware.race.mom.serializer; + +import java.io.ByteArrayInputStream; +import java.io.ObjectInputStream; +import java.util.List; + +import org.nustaq.serialization.FSTConfiguration; +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.serializers.JavaSerializer; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.ByteToMessageDecoder; + +public class RpcDecoder extends ByteToMessageDecoder { + + + private FSTConfiguration fst=FSTConfiguration.getDefaultConfiguration(); + @Override + public final void decode(ChannelHandlerContext ctx, ByteBuf in, List out) throws Exception { + try{ + if (in.readableBytes() < 4) { + return; + } + in.markReaderIndex(); + int dataLength = in.readInt(); + if (dataLength < 0) { + ctx.close(); + } + if (in.readableBytes() < dataLength) { + in.resetReaderIndex(); + return ; + } + byte[] data = new byte[dataLength]; + in.readBytes(data); + out.add(fst.asObject(data)); + + }catch(Exception e) + { + e.printStackTrace(); + } + + } +} + + + diff --git a/src/main/java/com/alibaba/middleware/race/mom/serializer/RpcEncoder.java b/src/main/java/com/alibaba/middleware/race/mom/serializer/RpcEncoder.java new file mode 100644 index 0000000..36a349a --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/serializer/RpcEncoder.java @@ -0,0 +1,48 @@ + +package com.alibaba.middleware.race.mom.serializer; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectOutputStream; +import java.nio.ByteBuffer; + +import org.nustaq.serialization.FSTConfiguration; +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; +import com.esotericsoftware.kryo.serializers.JavaSerializer; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.MessageToByteEncoder; +import io.netty.util.CharsetUtil; + +public class RpcEncoder extends MessageToByteEncoder { + + private FSTConfiguration fst=FSTConfiguration.getDefaultConfiguration(); + public RpcEncoder() { + + } + @Override + public void encode(ChannelHandlerContext ctx, Object in, ByteBuf out) throws Exception { + try{ + //对象 + if (!String.class.isInstance(in)) { + byte[] data =fst.asByteArray(in); + //写入内容 + out.writeInt(data.length); + out.writeBytes(data); + + } + //字符串 + else + { + String data=in.toString(); + out.writeBytes(data.getBytes()); + } + }catch(Exception e) + { + e.printStackTrace(); + } + } +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/store/ArrayStore.java b/src/main/java/com/alibaba/middleware/race/mom/store/ArrayStore.java new file mode 100644 index 0000000..ab66204 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/store/ArrayStore.java @@ -0,0 +1,32 @@ +package com.alibaba.middleware.race.mom.store; + +import java.util.ArrayList; + + + +/** + * 数组存储与恢复 + * @param + * + */ +public interface ArrayStore { + /** + * 数组存储 + * 每次调用以更新的方式（不能追加） + * @param offsets 需要存储的OffsetNum [] （一般用Integer[]），存储OffsetNum [] 中所有条目 + * @param topicAndFilter 这几个参数用于文件夹结构，和消息存储一样（去掉了consumerid项） + * @param groupId + * @param queueId + */ + public boolean store(ArrayList arrayList,String topicAndFilter, String queueId); + /** + * 数组恢复 + * @param topicAndFilter 这几个参数用于文件夹结构，和消息存储一样（去掉了consumerid项） + * @param groupId + * @param queueId + * @return OffsetNum []，读出所有条目，并装到 OffsetNum []（一般用Integer[]）返回 + */ + public ArrayList recover(String topicAndFilter, String queueId); + + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/store/ArrayStoreImp.java b/src/main/java/com/alibaba/middleware/race/mom/store/ArrayStoreImp.java new file mode 100644 index 0000000..fe33bb8 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/store/ArrayStoreImp.java @@ -0,0 +1,55 @@ +package com.alibaba.middleware.race.mom.store; + +import java.util.ArrayList; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.fastjson.JSON; + +public class ArrayStoreImp implements ArrayStore { + private static String filePath = StoreConfig.baseDir + StoreConfig.arrayDir; + private static Logger logger = LoggerFactory.getLogger(ArrayStoreImp.class); + private static ConcurrentHashMap fileStoreManagerMap = new ConcurrentHashMap(); + + @Override + public boolean store(ArrayList arrayList, String topicAndFilter, + String queueId) { + // TODO Auto-generated method stub + // 对应的文件管理器 + String arrayFilePath = filePath + "/" + topicAndFilter+"/"+queueId; + FileStoreManager fileStoreManager = fileStoreManagerMap + .get(arrayFilePath); + if (fileStoreManager == null) { + fileStoreManager = new FileStoreManager(arrayFilePath); + // 为恢复 + fileStoreManagerMap.put(arrayFilePath, fileStoreManager); + } + // 将写位置重置为16 + fileStoreManager.resetFilePosition(); + + ArrayList dataList = new ArrayList(arrayList.size()); + int count = 1; + for (OffsetNum tmp : arrayList) { + logger.debug("保存订阅关系 i+== " + (count++) + " , " + tmp.i); + dataList.add(JSON.toJSONBytes(tmp)); + } + + return fileStoreManager.saveMessage(dataList); + } + + @Override + public ArrayList recover(String topicAndFilter, String queueId) { + String arrayFilePath = filePath + "/" + topicAndFilter+"/"+queueId; + FileStoreManager fileStoreManager = fileStoreManagerMap + .get(arrayFilePath); + if (fileStoreManager == null) { + fileStoreManager = new FileStoreManager(arrayFilePath); + // 为恢复 + fileStoreManagerMap.put(arrayFilePath, fileStoreManager); + } + return (ArrayList) fileStoreManager.getOffsetNum(); + } + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/store/Comm.java b/src/main/java/com/alibaba/middleware/race/mom/store/Comm.java new file mode 100644 index 0000000..95fddb9 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/store/Comm.java @@ -0,0 +1,40 @@ +package com.alibaba.middleware.race.mom.store; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Comm { + /** + * 获取每个文件的名称地址 + * + */ + private static Logger logger = LoggerFactory + .getLogger(Comm.class); + + + public static ArrayList getFileNum(String filePath) { + ArrayList fileNumArray=new ArrayList(); + logger.error("filePath==="+filePath); + File indexFile = new File(filePath); + if (indexFile.isDirectory()) { + String[] fileList = indexFile.list(); + int fileCount = fileList.length; + if (fileCount == 0) { + logger.warn(" 没有 index file 存在！"); + } else { + // 遍历文件编号并保存 + for (int i = 0; i < fileCount; i++) { + int t = Integer.parseInt(fileList[i]); + fileNumArray.add(t); + logger.error(" 查找index目录下的所有文件并记录为一个数组中 fileList[i]= "+fileList[i]); + } + } + Collections.sort(fileNumArray); + } + return fileNumArray; + } +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/store/DataMappedFile.java b/src/main/java/com/alibaba/middleware/race/mom/store/DataMappedFile.java new file mode 100644 index 0000000..3dd2d18 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/store/DataMappedFile.java @@ -0,0 +1,446 @@ +package com.alibaba.middleware.race.mom.store; + +import java.io.File; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DataMappedFile { + // 文件编号 + private long dataNum; + // 文件大小 + private long fileSize; + // 文件数目 + private long fileCount = 0; + // data文件 + private MappedFileInfo dataMappedFile; + // 文件的fileChannel + private FileChannel fileChannel; + // 文件的MappedByteBuffer + private MappedByteBuffer mappedByteBuffer; + // 文件夹的路径 + private String filePath; + private static Logger logger = LoggerFactory + .getLogger(DataMappedFile.class); + // 保存已经打开的读文件列表 + private static ConcurrentHashMap readDataFileMap = new ConcurrentHashMap(); + public long dataWriterPositon; + + /** + * 构造函数 + * + * @param filePath + */ + public DataMappedFile(String filePath) { + this.filePath = filePath; + this.initFileInfo(); + /* + * this.mappedByteBuffer = this.dataMappedFile.getMappedByteBuffer(); + * this.fileChannel = this.dataMappedFile.getFileChannel(); + */ + + } + + /** + * 把数据从data文件的读出 带msg大小的标识 + * + * @param Offset + * 在data文件中的位置 + * @param size + * 写入data文件中数据的大小 + * @param dataFileNum + * 写入data文件的名称 + * @return + * + */ + public List checkLastMsg(long[] indexData) { + // 保存已经开打的再读data文件 + List rs = new ArrayList(); + int offset, size, dataFileNum; + if (indexData == null) { + // 初始状态，index文件没有数据 + logger.error("indexData == null"); + dataFileNum = 0; + offset = 16; + size = 0; + logger.error(" int offset, int size, int dataFileNum=" + offset + + " " + size + " " + dataFileNum); + } else { + offset = (int) indexData[0]; + size = (int) indexData[1]; + dataFileNum = (int) indexData[2]; + logger.error("int offset, int size, int dataFileNum=" + offset + + " " + size + " " + dataFileNum); + + } + MappedFileInfo readForDataFile = readDataFileMap.get(this.filePath + + "/" + dataFileNum); + if (readForDataFile == null) { + readForDataFile = new MappedFileInfo(this.filePath + "/" + + dataFileNum); + readDataFileMap.put(this.filePath + "/" + dataFileNum, + readForDataFile); + } + + MappedByteBuffer readMappedByteFile = readForDataFile + .getMappedByteBuffer(); + int writePos = (int) readForDataFile.getWritePosition().get(); + logger.error("int writePos = (int) readForDataFile.getWritePosition().get();=" + + writePos); + // 定位到读位置，然后获取byte数据 + int currentPos = offset + size + 4; + // 当前data文件中 + if (currentPos < writePos) { + do { + readMappedByteFile.position(currentPos); + long nextSize = readMappedByteFile.getInt(); + long[] t = new long[3]; + t[0] = currentPos; + t[1] = nextSize; + t[2] = dataFileNum; + logger.error("t[0] = currentPos;t[1] = nextSize;t[2]=dataFileNum;" + + t.toString()); + rs.add(t); + currentPos += nextSize + 4; + } while (currentPos < writePos); + // 判断是否还有下一个data文件,并进行恢复 + ArrayList fileNameList = Comm.getFileNum(this.filePath); + for (int i=0; i < fileNameList.size(); i++) { + if(fileNameList.get(i)>dataFileNum){ + currentPos=16; + dataFileNum=fileNameList.get(i); + do { + readMappedByteFile.position(currentPos); + long nextSize = readMappedByteFile.getInt(); + long[] t = new long[3]; + t[0] = currentPos; + t[1] = nextSize; + t[2] = dataFileNum; + logger.error("t[0] = currentPos;t[1] = nextSize;t[2]=dataFileNum;" + + t.toString()); + rs.add(t); + currentPos += nextSize + 4; + } while (currentPos < writePos); + } + } + } + return rs; + } + + /** + * 获取每个文件的名称地址 + * + */ + private ArrayList dataNumArray; + + /** + * 创建或者获取当前使用的index文件 暂时以文件的数目来确定存储文件 + */ + private void initFileInfo() { + dataNumArray = Comm.getFileNum(this.filePath); + if (this.dataNumArray == null || this.dataNumArray.size() == 0) { + logger.warn("create index file"); + this.dataMappedFile = new MappedFileInfo(this.filePath + "/0"); + this.dataNum = 0; + this.fileCount = 1; + this.dataNumArray.add(0); + } else { + // 判断文件的最大编号 + int max = dataNumArray.get(dataNumArray.size() - 1); + logger.warn("max index " + max); + this.dataNum = max; + this.dataMappedFile = new MappedFileInfo(this.filePath + "/" + + this.dataNum); + this.fileCount = dataNumArray.size(); + } + + logger.error("initFileInfo 创建了一个datafile + this.datanum===" + + this.dataNum); + readDataFileMap.put(this.filePath + "/" + this.dataNum, + this.dataMappedFile); + } + + /** + * 保存data数据到data文件 + * + * @param data + * @return + */ + public long addData(byte[] data) { + // 保存当写入失败是上次写的位置 + long lastWritePosition = this.dataMappedFile.getWritePosition().get(); + boolean addDataFlag = this.dataMappedFile.saveData(data); + if (!addDataFlag) { + logger.error("增加数据时，当前data文件空间不够 ,将新建data文件"); + // 恢复data的写位置 + this.dataMappedFile.setWritePosition(new AtomicLong( + lastWritePosition)); + this.dataMappedFile.setWritePositionToFile(lastWritePosition); + this.dataMappedFile.flush(); + // 新建data文件 + this.dataMappedFile = new MappedFileInfo(this.filePath + "/" + + this.fileCount); + this.dataNum = this.fileCount; + dataNumArray.add((int) this.dataNum); + Collections.sort(dataNumArray); + this.fileCount++; + // 重新写入文件 + return this.addData(data); + } + return lastWritePosition; + // this.dataMappedFile.flush(); + } + + /** + * 保存data数据到data文件 + * + * @param data + * @return + */ + public long addData_new(byte[] data) { + // 保存当写入失败是上次写的位置 + long lastWritePosition = this.dataMappedFile.getWritePosition().get(); + // 新存储data + boolean addDataFlag = this.dataMappedFile.saveData(data); + if (!addDataFlag) { + logger.error("增加数据时，当前data文件空间不够 将新建data文件"); + // 恢复data的写位置 + this.dataMappedFile.setWritePosition(new AtomicLong( + lastWritePosition)); + this.dataMappedFile.setWritePositionToFile(lastWritePosition); + this.dataMappedFile.flush(); + // 新建data文件 + this.dataMappedFile = new MappedFileInfo(this.filePath + "/" + + this.fileCount); + this.dataNum = this.fileCount; + dataNumArray.add((int) this.dataNum); + Collections.sort(dataNumArray); + this.fileCount++; + // 重新写入文件 + return this.addData(data); + } + return lastWritePosition; + // this.dataMappedFile.flush(); + } + + + /** + * 保存dataList数据到data文件 + * + * @param data + * @return + */ + public long[] addDataByOnce(List dataList) { + // 保存当写入失败是上次写的位置 + long lastWritePosition = this.dataMappedFile.getWritePosition().get(); + long[] result = this.dataMappedFile.saveDataByOnce(dataList); + + if (result == null) { + logger.error("增加数据时，当前data文件空间不够 将新建data文件"); + // 恢复data的写位置 + this.dataMappedFile.setWritePosition(new AtomicLong( + lastWritePosition)); + this.dataMappedFile.setWritePositionToFile(lastWritePosition); + this.dataMappedFile.flush(); + // 新建data文件 + this.dataMappedFile = new MappedFileInfo(this.filePath + "/" + + this.fileCount); + this.dataNum = this.fileCount; + // 加入数据并排序 + dataNumArray.add((int) this.dataNum); + Collections.sort(dataNumArray); + this.fileCount++; + // 重新写入文件 + logger.error("增加数据时，当前data文件空间不够 将新建data文件" + dataNum); + return this.addDataByOnce(dataList); + } + return result; + // this.dataMappedFile.flush(); + } + + /** + * 将文件buffer刷新落盘保持到磁盘 + */ + public void flush() { + this.dataMappedFile.flush(); + } + + /** + * 从指定文件中获取byte[]数据 + * + * @param offset + * 指定文件读取位置 size 读取大小 dataFileName 数据文件 + * @return byte[]数据 + */ + + public byte[] getData(long[] dataPos) { + return getData((int) dataPos[0], (int) dataPos[1], (int) dataPos[2]); + } + + /** + * 获取一组数据 + * + */ + public List getDataByOnce(List dataIndexList) { + List result = new ArrayList(); + for (int i = 0; i < dataIndexList.size(); i++) { + long[] dataPos = dataIndexList.get(i); + result.add(getData((int) dataPos[0], (int) dataPos[1], + (int) dataPos[2])); + } + return result; + + } + + /** + * 把数据从data文件的读出 + * + * @param Offset + * 在data文件中的位置 + * @param size + * 写入data文件中数据的大小 + * @param dataFileNum + * 写入data文件的名称 + * @return + * + */ + public byte[] getData(int offset, int size, int dataFileNum) { + // 保存已经开打的再读data文件 + MappedFileInfo readForDataFile = readDataFileMap.get(this.filePath + + "/" + dataFileNum); + if (readForDataFile == null) { + readForDataFile = new MappedFileInfo(this.filePath + "/" + + dataFileNum); + readDataFileMap.put(this.filePath + "/" + dataFileNum, + readForDataFile); + } + + MappedByteBuffer readMappedByteFile = readForDataFile + .getMappedByteBuffer(); + // 定位到读位置，然后获取byte数据 + readMappedByteFile.position(offset + 4); + byte[] tempData = new byte[size]; + readMappedByteFile.get(tempData); + return tempData; + } + + /** + * 把数据从data文件的读出 带msg大小的标识 + * + * @param Offset + * 在data文件中的位置 + * @param size + * 写入data文件中数据的大小 + * @param dataFileNum + * 写入data文件的名称 + * @return + * + */ + public byte[] getData_new(int offset, int size, int dataFileNum) { + // 保存已经开打的再读data文件 + System.out.println("int offset, int size, int dataFileNum=" + offset + + " " + size + " " + dataFileNum); + MappedFileInfo readForDataFile = readDataFileMap.get(this.filePath + + "/" + dataFileNum); + if (readForDataFile == null) { + readForDataFile = new MappedFileInfo(this.filePath + "/" + + dataFileNum); + readDataFileMap.put(this.filePath + "/" + dataFileNum, + readForDataFile); + } + + MappedByteBuffer readMappedByteFile = readForDataFile + .getMappedByteBuffer(); + // 定位到读位置，然后获取byte数据 + readMappedByteFile.position(offset + 4); + byte[] tempData = new byte[size]; + readMappedByteFile.get(tempData); + return tempData; + } + + /** + * 获取从读位置到写位置的所有的数据 + * + * @return + */ + public byte[] getData() { + return this.dataMappedFile.getData(); + + } + + public long getFileSize() { + return fileSize; + } + + public void setFileSize(long fileSize) { + this.fileSize = fileSize; + } + + public long getFileCount() { + return fileCount; + } + + public void setFileCount(long fileCount) { + this.fileCount = fileCount; + } + + public MappedFileInfo getDataMappedFile() { + return dataMappedFile; + } + + public void setDataMappedFile(MappedFileInfo dataMappedFile) { + this.dataMappedFile = dataMappedFile; + } + + public FileChannel getFileChannel() { + return fileChannel; + } + + public void setFileChannel(FileChannel fileChannel) { + this.fileChannel = fileChannel; + } + + public MappedByteBuffer getMappedByteBuffer() { + return mappedByteBuffer; + } + + public void setMappedByteBuffer(MappedByteBuffer mappedByteBuffer) { + this.mappedByteBuffer = mappedByteBuffer; + } + + public String getFilePath() { + return filePath; + } + + public void setFilePath(String filePath) { + this.filePath = filePath; + } + + public long getDataNum() { + return dataNum; + } + + public void setDataNum(long dataNum) { + this.dataNum = dataNum; + } + + public void destory() { + // TODO Auto-generated method stub + // this.mappedByteBuffer + } + + /** + * 重新设置写位置 + */ + public void resetFilePosition() { + this.dataMappedFile.setWritePositionToFile(16); + this.dataMappedFile.setWritePosition(new AtomicLong(16)); + } +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/store/FileStoreManager.java b/src/main/java/com/alibaba/middleware/race/mom/store/FileStoreManager.java new file mode 100644 index 0000000..4ea9c2d --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/store/FileStoreManager.java @@ -0,0 +1,330 @@ +package com.alibaba.middleware.race.mom.store; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.fastjson.JSON; +import com.alibaba.middleware.race.mom.broker.function.Offset; + +public class FileStoreManager { + private static Logger logger = LoggerFactory + .getLogger(FileStoreManager.class); + // 索引文件 + private IndexMappedFile indexMappedFile; + // 数据文件 + private DataMappedFile dataMappedFile; + + private String filePath; + // 最后一次刷数据的时间 + private long lastFlushTime = System.currentTimeMillis(); + private long lastIndexFlushTime = System.currentTimeMillis(); + // 读写锁保证线程安全 + private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + // 统计已经写入但为落盘到磁盘的消息数目 + private static AtomicLong countMsg = new AtomicLong(0); + // 落盘时，同步等待index和data文件的缓存落盘到磁盘 + private CountDownLatch countDownLatch; + // index和data文件多线程并行落盘 + private ExecutorService flushPool = Executors.newFixedThreadPool(1); + private ExecutorService flushIndexPool = Executors.newFixedThreadPool(1); + + public FileStoreManager(String filePath) { + this.filePath = filePath; + this.indexMappedFile = new IndexMappedFile(filePath + "/index"); + this.dataMappedFile = new DataMappedFile(filePath + "/data"); + // logger.debug("初始化filestoremanager成功！" + filePath); + } + + public FileStoreManager(String filePath, String test) { + this.filePath = filePath; + this.indexMappedFile = new IndexMappedFile(filePath + "/index"); + this.dataMappedFile = new DataMappedFile(filePath + "/data"); + // logger.debug("初始化filestoremanager成功！" + filePath); + this.recover(); + } + + void recover() { + long[] lastWrittePosition = this.indexMappedFile.getLastWriteData(); + // logger.error("long[] lastWrittePosition=="+lastWrittePosition.toString()); + List rs = this.dataMappedFile.checkLastMsg(lastWrittePosition); + if (rs == null || rs.size() == 0) { + return; + } else { + for (long[] temp : rs) { + logger.error("新恢复增加indexdata " + temp[0] + " / " + temp[1] + + " / " + temp[2]); + this.indexMappedFile.addData(temp[0], (int) temp[1], + (int) temp[2]); + } + } + } + + /** + * 保存一个字节到index和data文件中 + * + * @param msg + */ + private void saveMessage(byte[] msg) { + lock.writeLock().lock(); + try { + // 在data文件中增加记录 + int dataWritePosition = (int) this.dataMappedFile.addData(msg); + int size = msg.length; + int fileNum = (int) this.dataMappedFile.getDataNum(); + // logger.debug("dataWritePosition,size,fileNum= " + + // dataWritePosition + // + "," + size + "," + fileNum); + + // 写入index文件中 + boolean flag = this.indexMappedFile.addData(dataWritePosition, + size, fileNum); + if (!flag) { + logger.warn("没有存成功，未知原因，在index文件中"); + } + logger.debug("成功将字节写入文件"); + } finally { + lock.writeLock().unlock(); + } + } + + /** + * 保存数据 提供给其他调用的接口，必须是以List包装的字节数据，传入的参数最好用ArrayList + */ + static long savemessage = 0; + + public boolean saveMessage(List msgList) { + long t1 = System.currentTimeMillis(); + for (int i = 0; i < msgList.size(); i++) { + byte[] temp = msgList.get(i); + countMsg.addAndGet(1); + saveMessage(temp); + } + + flush(); + long t2 = System.currentTimeMillis(); + // savemessage += (t2 - t1); + logger.error("flush time t2-t1= " + (t2 - t1)); + return true; + } + + /** + * 保存数据 提供给其他调用的接口，必须是以List包装的字节数据，传入的参数最好用ArrayList + */ + + static long savemessagebyonce = 0; + + public boolean saveMessageByOnce(List msgList) { + long t1 = System.currentTimeMillis(); + + // 获取data文件存储结果 + logger.debug("开始组进入写data文件 msgList.size()==" + msgList.size()); + long[] dataResult = this.dataMappedFile.addDataByOnce(msgList); + + long dataFileNum = this.dataMappedFile.getDataNum(); + logger.debug("开始进入写index文件 dataResult.length= " + dataResult.length); + + boolean indexFlag = this.indexMappedFile.addDataByOnce(dataResult, + (int) dataFileNum); + + flush(); + long t2 = System.currentTimeMillis(); + logger.error("flush time t2-t1= " + (t2 - t1)); + // savemessagebyonce += (t2 - t1); + return indexFlag; + } + + /** + * 获取第0条到写的位置条数据, 恢复订阅数据用的函数 + * + * @return + */ + public List getSubscribe() { + ByteBuffer indexByte = ByteBuffer.wrap(this.indexMappedFile.getData()); + ByteBuffer dataByte = ByteBuffer.wrap(this.dataMappedFile.getData()); + int count = indexByte.capacity() / 16; + List subscribeInfoList = new ArrayList(count); + for (int i = 0; i < count; i++) { + indexByte.position(i * 16); + long dataStart = indexByte.getLong(); + int size = indexByte.getInt(); + dataByte.position((int) dataStart - 16 + 4); + byte[] temp = new byte[size]; + dataByte.get(temp); + subscribeInfoList + .add((Offset) JSON.parseObject(temp, Offset.class)); + } + return subscribeInfoList; + } + + /** + * 获取第startCount条到endCount条数据 采用一组数据一组数据的读 + * + * @param startCount + * @param endCount + * @return + */ + public ArrayList getMessageByOnce(int offset, int maxOffset) { + logger.warn("开始组获取数据了"); + List dataIndexList = this.indexMappedFile.getIndexDataByOnce( + offset, maxOffset); + return (ArrayList) this.dataMappedFile + .getDataByOnce(dataIndexList); + + } + + /** + * 获取第startCount条到endCount条数据 + * + * @param startCount + * @param endCount + * @return + */ + public List getMessage(int startCount, int endCount) { + lock.readLock().lock(); + List msgList = null; + try { + msgList = new ArrayList(endCount - startCount + 1); + // logger.debug(" return msgList.size() endCount - startCount + 1 = " + // + (endCount - startCount + 1)); + for (int i = startCount; i <= endCount; i++) { + // logger.debug("i= " + i); + long[] returnIndexData = this.indexMappedFile.getData(i); + // logger.debug(" returnIndexData= " + returnIndexData[0] + + // " , " + // + returnIndexData[1] + " , " + returnIndexData[2] + " , "); + byte[] temp = this.dataMappedFile.getData(returnIndexData); + // logger.debug("temp.size()= " + temp.length); + msgList.add(temp); + } + } catch (Exception e) { + } finally { + lock.readLock().unlock(); + } + return msgList; + } + + /** + * 多线程并行落盘 + */ + public void flush() { + lock.writeLock().lock(); + countDownLatch = new CountDownLatch(2); + + flushIndexPool.execute(new Thread() { + @Override + public void run() { + long t1 = System.currentTimeMillis(); + + // 索引文件异步 + try { + FileStoreManager.this.indexMappedFile.flush(); + countDownLatch.countDown(); + } finally { + // countDownLatch.countDown(); + } + long t2 = System.currentTimeMillis(); + // System.out.println(" force index t2-t1==" + (t2 - t1)); + } + }); + + flushPool.execute(new Thread() { + @Override + public void run() { + long t1 = System.currentTimeMillis(); + try { + FileStoreManager.this.dataMappedFile.flush(); + } finally { + countDownLatch.countDown(); + } + long t2 = System.currentTimeMillis(); + // System.out.println(" force data t2-t1==" + (t2 - t1)); + } + }); + + try { + countDownLatch.await(); + } catch (Exception e) { + logger.error(e.getMessage()); + } finally { + lock.writeLock().unlock(); + } + + // 测试用的 + // flushPool.shutdown(); + } + + /** + * 多线程并行落盘 + */ + /* + * public void flush() { lock.writeLock().lock(); countDownLatch = new + * CountDownLatch(2); + * + * flushIndexPool.execute(new Thread() { + * + * @Override public void run() { long t1 = System.currentTimeMillis(); + * + * // 索引文件异步 try { FileStoreManager.this.indexMappedFile.flush(); + * countDownLatch.countDown(); } finally { // countDownLatch.countDown(); } + * long t2 = System.currentTimeMillis(); // + * System.out.println(" force index t2-t1==" + (t2 - t1)); } }); + * + * flushPool.execute(new Thread() { + * + * @Override public void run() { long t1 = System.currentTimeMillis(); try { + * FileStoreManager.this.dataMappedFile.flush(); } finally { + * countDownLatch.countDown(); } long t2 = System.currentTimeMillis(); // + * System.out.println(" force data t2-t1==" + (t2 - t1)); } }); + * + * try { countDownLatch.await(); } catch (Exception e) { + * logger.error(e.getMessage()); } finally { lock.writeLock().unlock(); } + * + * // 测试用的 // flushPool.shutdown(); } + */ + + /** + * 释放资源 + */ + public void destory() { + this.indexMappedFile.destory(); + this.dataMappedFile.destory(); + } + + public void resetFilePosition() { + this.indexMappedFile.resetFilePosition(); + this.dataMappedFile.resetFilePosition(); + } + + /** + * 获取第0条到写的位置条数据, 恢复OffsetNum数据用的函数 + * + * @return + */ + public List getOffsetNum() { + ByteBuffer indexByte = ByteBuffer.wrap(this.indexMappedFile.getData()); + ByteBuffer dataByte = ByteBuffer.wrap(this.dataMappedFile.getData()); + int count = indexByte.capacity() / 16; + List offsetNumList = new ArrayList(count); + for (int i = 0; i < count; i++) { + indexByte.position(i * 16); + long dataStart = indexByte.getLong(); + int size = indexByte.getInt(); + dataByte.position((int) dataStart - 16 + 4); + byte[] temp = new byte[size]; + dataByte.get(temp); + offsetNumList.add((OffsetNum) JSON.parseObject(temp, + OffsetNum.class)); + } + return offsetNumList; + } + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/store/IndexMappedFile.java b/src/main/java/com/alibaba/middleware/race/mom/store/IndexMappedFile.java new file mode 100644 index 0000000..a0b6505 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/store/IndexMappedFile.java @@ -0,0 +1,296 @@ +package com.alibaba.middleware.race.mom.store; + +import java.io.File; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class IndexMappedFile { + // 文件编号 + private long indexNum; + // 文件大小 + private long fileSize; + // 文件数目 + private long fileCount = 0; + // index文件 + private MappedFileInfo indexMappedFile; + + // 文件的fileChannel + private FileChannel fileChannel; + // 文件的MappedByteBuffer + private MappedByteBuffer mappedByteBuffer; + // 文件夹的路径 + private String filePath; + // 保存已经打开的读文件列表 + ConcurrentHashMap readIndexFileMap = new ConcurrentHashMap(); + + private static Logger logger = LoggerFactory + .getLogger(IndexMappedFile.class); + + public long[] getLastWriteData() { + long[] result=null; + long pos = this.indexMappedFile.getWritePosition().get(); + if (pos == 16) { + //初始状态 + return result; + } + this.indexMappedFile.getMappedByteBuffer().position((int) (this.indexMappedFile + .getWritePosition().get() - 16)); + long offset = this.indexMappedFile.getMappedByteBuffer().getLong(); + + logger.error("从index 读出data 的位置offset====" + offset); + int size = this.indexMappedFile.getMappedByteBuffer().getInt(); + int dataFileNum = this.indexMappedFile.getMappedByteBuffer().getInt(); + result = new long[] { offset, size, dataFileNum }; + return result; + } + + /** + * 构造函数 + * + * @param filePath + */ + public IndexMappedFile(String filePath) { + this.filePath = filePath; + this.initFileInfo(); + /* + * this.mappedByteBuffer = this.indexMappedFile.getMappedByteBuffer(); + * this.fileChannel = this.indexMappedFile.getFileChannel(); + */ + } + + /** + * 获取每个文件的名称地址 + * + */ + private ArrayList indexNumArray; + + + /** + * 创建或者获取当前使用的index文件 暂时以文件的数目来确定存储文件 + */ + private void initFileInfo() { + indexNumArray = Comm.getFileNum(this.filePath); + if (this.indexNumArray == null || this.indexNumArray.size() == 0) { + logger.warn("create index file"); + this.indexMappedFile = new MappedFileInfo(this.filePath + "/0"); + this.indexNum = 0; + this.fileCount = 1; + this.indexNumArray.add(0); + } else { + // 判断文件的最大编号 + int max = indexNumArray.get(indexNumArray.size() - 1); + logger.error("max index " + max); + this.indexNum = max; + this.indexMappedFile = new MappedFileInfo(this.filePath + "/" + + this.indexNum); + this.fileCount = indexNumArray.size(); + } + + logger.error("inti file createFile 创建了一个indexfile + this.indexnum===" + + this.indexNum); + readIndexFileMap.put(this.filePath + "/" + this.indexNum, + this.indexMappedFile); + } + + + /** + * 把数据在data文件的写入信息保存到index文件中 + * + * @param dataOffset + * 在data文件中的位置 + * @param size + * 写入data文件中数据的大小 + * @param dataFileNum + * 写入data文件的名称 + * @return + */ + public boolean addData(long dataOffset, int size, int dataFileNum) { + // 保存当写入失败是上次写的位置 + long lastWritePosition = this.indexMappedFile.getWritePosition().get(); + // 依次写入data数据信息 + boolean[] addDataFlag = new boolean[3]; + addDataFlag[0] = this.indexMappedFile.saveData(dataOffset); + addDataFlag[1] = this.indexMappedFile.saveData(size); + addDataFlag[2] = this.indexMappedFile.saveData(dataFileNum); + if (!(addDataFlag[0] && addDataFlag[1] && addDataFlag[2])) { + logger.error("增加数据时，当前index文件空间不够 将新建index文件"); + // 恢复index的写位置 + this.indexMappedFile.setWritePosition(new AtomicLong( + lastWritePosition)); + this.indexMappedFile.setWritePositionToFile(lastWritePosition); + this.indexMappedFile.flush(); + // 新建index文件 + // 新建index文件,文件名为开始记录 + logger.error("index 增加数据时，当前index文件空间不够 将新建index文件 old this.indexNum=" + + this.indexNum + + " /lastWritePosition==" + + lastWritePosition); + this.indexNum += ((lastWritePosition / 16) - 1); + logger.error("index 增加数据时，当前index文件空间不够 将新建index文件 new this.indexNum=" + + this.indexNum); + // 加入数组中，默认是加到数据末尾 + indexNumArray.add((int) this.indexNum); + this.indexMappedFile = new MappedFileInfo(this.filePath + "/" + + this.indexNum); + logger.error("index 增加数据时，当前index文件空间不够 将新建index文件 his.indexNum" + + this.indexNum); + this.fileCount++; + // 重新写入文件 + return this.addData(dataOffset, size, dataFileNum); + } + logger.debug("增加index数据时 成功"); + return true; + } + + /** + * 把数据在data文件的写入信息保存到index文件中 + * + * @param dataOffset + * 在data文件中的位置 + * @param size + * 写入data文件中数据的大小 + * @param dataFileNum + * 写入data文件的名称 + * @return + */ + public boolean addDataByOnce(long[] dataArray, int dataFileNum) { + + // 保存当写入失败是上次写的位置 + long lastWritePosition = this.indexMappedFile.getWritePosition().get(); + // 依次写入data数据信息 + boolean addDataFlag = this.indexMappedFile.saveDataByOnce(dataArray, + dataFileNum); + if (!addDataFlag) { + logger.error("增加数据时，当前index文件空间不够 将新建index文件"); + // 恢复index的写位置 + this.indexMappedFile.setWritePosition(new AtomicLong( + lastWritePosition)); + this.indexMappedFile.setWritePositionToFile(lastWritePosition); + this.indexMappedFile.flush(); + // 新建index文件,文件名为开始记录 + this.indexNum += ((lastWritePosition / 16) - 1); + // 加入数组中，默认是加到数据末尾 + indexNumArray.add((int) this.indexNum); + + logger.error("create index file name indexnum== " + this.indexNum); + logger.error("index 增加数据时，当前index文件空间不够 将新建index文件 indexNumArray.SIZE()" + + indexNumArray.size()); + this.indexMappedFile = new MappedFileInfo(this.filePath + "/" + + this.indexNum); + this.fileCount++; + // 重新写入文件 + return this.addDataByOnce(dataArray, dataFileNum); + } + logger.debug("增加index数据时 成功"); + return true; + // this.indexMappedFile.flush(); + } + + /** + * 将文件buffer落盘保持到磁盘 + */ + public long flush() { + return this.indexMappedFile.flush(); + } + + /** + * 获取信息，防护给data文件读取数据， 获取指定位置的一个单位(8+4+4) + * + */ + public long[] getData(int countStart) { + // 获取需要打开index文件名 + if (indexNumArray == null || indexNumArray.size() == 0) { + logger.error(" 没有 index file 存在！countStart= " + countStart); + return null; + } else { + // 判断文件的最大编号 + int fileCount = indexNumArray.size(); + int readIndexFileNum = 0; + for (int i = (fileCount - 1); i >= 0; i--) { + // logger.debug("get data create index filedir count tempNum= " + // + readIndexFileNum); + int t = indexNumArray.get(i); + if (t > countStart) { + continue; + } else { + readIndexFileNum = t; + break; + } + } + // 转化到当前文件的文件内地址 + + logger.debug("转化到当前文件的文件内地址 readIndexFileNum___" + readIndexFileNum); + MappedFileInfo readForIndexFile = readIndexFileMap + .get(this.filePath + "/" + readIndexFileNum); + + if (readForIndexFile == null) { + readForIndexFile = new MappedFileInfo(this.filePath + "/" + + readIndexFileNum); + readIndexFileMap.put(this.filePath + "/" + readIndexFileNum, + readForIndexFile); + logger.error("创建读index文件 ：" + this.filePath + "/" + + readIndexFileNum); + } + // logger.debug("读index文件 ：" + this.filePath + "/" + fileNum); + // 将countStart转化为指定index文件的相对地址，注意要偏移16位 + int tempPosition = (countStart - readIndexFileNum) * 16 + 16; + MappedByteBuffer readMappedByteFile = readForIndexFile + .getMappedByteBuffer(); + readMappedByteFile.position((int) tempPosition); + logger.error("在读出index 数据的 位置 tempPosition====" + tempPosition); + // 获取与data文件记录相关信息 + long offset = readMappedByteFile.getLong(); + logger.error("从index 读出data 的位置offset====" + offset); + int size = readMappedByteFile.getInt(); + int dataFileNum = readMappedByteFile.getInt(); + long[] result = new long[] { offset, size, dataFileNum }; + return result; + } + + } + + /** + * 获取信息，防护给data文件读取数据， 获取指定位置的一个单位(8+4+4) + * + */ + public List getIndexDataByOnce(int countStart, int countEnd) { + List result = new ArrayList(); + for (int i = countStart; i <= countEnd; i++) { + logger.error("int i = countStart; i <= countEnd" + i + + " int countStart, int countEnd== " + countStart + " , " + + countEnd); + result.add(this.getData(i)); + } + return result; + } + + /** + * 获取从读位置到写位置的所有的数据 + * + * @return + */ + public byte[] getData() { + return this.indexMappedFile.getData(); + + } + + public void destory() { + // this.indexMappedFile + } + + /** + * 重新设置写位置 + */ + public void resetFilePosition() { + this.indexMappedFile.setWritePositionToFile(16); + this.indexMappedFile.setWritePosition(new AtomicLong(16)); + } +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/store/MappedFileInfo.java b/src/main/java/com/alibaba/middleware/race/mom/store/MappedFileInfo.java new file mode 100644 index 0000000..90a36f7 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/store/MappedFileInfo.java @@ -0,0 +1,416 @@ +package com.alibaba.middleware.race.mom.store; + +import java.io.File; +import java.io.RandomAccessFile; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileChannel.MapMode; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MappedFileInfo { + // 文件大小 + private static long MAX_FILE_SIZE = 1024 * 1024 * 1024; + // 文件读写位置 + private AtomicLong writePosition = new AtomicLong(0); + private AtomicLong readPosition = new AtomicLong(0); + // 上一次刷的文件位置 + private AtomicLong lastFlushFilePosition = new AtomicLong(0); + // 最后一次刷数据的时间 + private long lastFlushTime = System.currentTimeMillis(); + + // 文件路径和名称全称 + private String filePathAndName; + // 文件的读取方式 + private RandomAccessFile raf; + private MappedByteBuffer mappedByteBuffer; + // 映射的FileChannel对象 + private FileChannel fileChannel; + private static Logger logger = LoggerFactory + .getLogger(MappedFileInfo.class); + + /** + * 构建一个mappedFile文件 + * + * @param filePathAndName + * 传入文件路径和名称 + */ + public MappedFileInfo(String filePathAndName) { + this.filePathAndName = filePathAndName; + createRandomAccessFile(); + getFileMappedByteBuffer(0, MAX_FILE_SIZE); + setInitPosition(); + } + + /** + * 构建一个mappedFile文件 + * + * @param filePathAndName + * 文件路径和名称 + * @param fileSize + * 文件大小 + */ + public MappedFileInfo(String filePathAndName, long fileSize) { + this.filePathAndName = filePathAndName; + this.createRandomAccessFile(); + this.getFileMappedByteBuffer(0, fileSize); + this.setInitPosition(); + } + + /** + * 初始化文件的读写位置变量 读位置（8）+写位置（8） 根据文件的前16个字节获取当前读和写的位置 + * + */ + private void setInitPosition() { + // 文件前16字节存放读地址和写地址 + this.mappedByteBuffer.position(0); + long position; + // 如果position=0则为起始地址16 + if ((position = this.mappedByteBuffer.getLong()) == 0) { + this.readPosition.set(16); + } else { + this.readPosition.set(position); + } + // 更新读位置 + this.mappedByteBuffer.position(0); + this.mappedByteBuffer.putLong(this.readPosition.get()); + // logger.debug("this.readPosition= "+this.readPosition); + // 如果position=0则为起始地址16 + if ((position = this.mappedByteBuffer.getLong()) == 0) { + this.writePosition.set(16); + this.lastFlushFilePosition.set(16); + } else { + this.writePosition.set(position); + this.lastFlushFilePosition.set(16); + } + // 更新写位置 + this.mappedByteBuffer.position(8); + this.mappedByteBuffer.putLong(this.writePosition.get()); + // logger.debug("this.writePosition= "+this.writePosition.get()); + } + + /** + * 建立和确保文件存在 + */ + private synchronized void createRandomAccessFile() { + File f = new File(filePathAndName); + try { + if (!f.exists()) { + if (!f.getParentFile().exists()) { + f.getParentFile().mkdirs(); + } + f.createNewFile(); + } + this.raf = new RandomAccessFile(f, "rw"); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + /** + * 打开文件映射 + * + * @param startPos + * @param fileSize + */ + private void getFileMappedByteBuffer(long startPos, long fileSize) { + try { + this.fileChannel = this.raf.getChannel(); + this.mappedByteBuffer = this.fileChannel.map(MapMode.READ_WRITE, + startPos, fileSize); + } catch (Exception e) { + logger.error("内存映射沒有建立"); + logger.error(e.getMessage()); + } + } + + /** + * 保存一组index数据 ，index文件调用 + * + * @param data + * @return + */ + public boolean saveDataByOnce(long[] dataArray, int fileNum) { + long lastposition = this.writePosition.get(); + long currentPosition = lastposition; + if ((currentPosition + (dataArray.length / 3) * 16) > this.MAX_FILE_SIZE) { + // 恢复到存数据的开始 + logger.warn("文件空间不够了，需要新建文件"); + return false; + } + for (int i = 0; i < dataArray.length; i++) { + // 一定要定位写的位置 + mappedByteBuffer.position((int) currentPosition); + mappedByteBuffer.putLong(dataArray[i]); + i++; + mappedByteBuffer.putInt((int) dataArray[i]); + i++; + mappedByteBuffer.putInt(fileNum); + currentPosition += 16; + } + // 完成后，设置写的写位置 + this.writePosition.set(currentPosition); + this.lastFlushFilePosition.set(currentPosition); + mappedByteBuffer.position(8); + mappedByteBuffer.putLong(currentPosition); + return true; + } + + /** + * 保存一组data数据 + * + * @param data + * @return + */ + public long[] saveDataByOnce(List dataList) { + long lastposition = this.writePosition.get(); + long currentPosition = lastposition; + int count = dataList.size(); + // 保存返回结果 + long[] result = new long[count * 3]; + for (int i = 0; i < count; i++) { + byte[] data = dataList.get(i); + if ((currentPosition + data.length + 4) > this.MAX_FILE_SIZE) { + // 恢复到存数据的开始 + logger.warn("保存消息的数据文件空间不够了！"); + currentPosition = lastposition; + result = null; + break; + } else { + // 表示有空余空间 + if (i == 0) { + // 加快速度 + mappedByteBuffer.position((int) currentPosition); + } + result[i * 3] = currentPosition; + result[3 * i + 1] = data.length; + mappedByteBuffer.putInt(data.length); + mappedByteBuffer.put(data); + currentPosition += data.length + 4; + } + } + // 设置写的写位置 + this.writePosition.set(currentPosition); + this.lastFlushFilePosition.set(currentPosition); + mappedByteBuffer.position(8); + mappedByteBuffer.putLong(currentPosition); + + return result; + } + + /** + * 保存单条数据 + * + * @param data + * @return + */ + public boolean saveData(byte[] data) { + long position = this.writePosition.get(); + logger.debug("save data[] this.writePosition.get() " + position); + // 表示有空余空间 + if ((position + data.length + 4) <= this.MAX_FILE_SIZE) { + mappedByteBuffer.position((int) position); + mappedByteBuffer.putInt(data.length); + mappedByteBuffer.put(data); + this.writePosition.addAndGet(data.length + 4); + this.lastFlushFilePosition.addAndGet(data.length + 4); + // 更新文件前16字节writePostion + mappedByteBuffer.position(8); + mappedByteBuffer.putLong(this.writePosition.get()); + return true; + } + return false; + } + + /** + * 保存单条数据 data文件 带长度位置的msg存储 + * + * @param data + * @return + */ + public boolean saveData_new(byte[] data) { + int length = data.length; + long position = this.writePosition.get(); + logger.error("save data[] this.writePosition.get() " + position); + logger.error("save data[] length() " + length); + // 表示有空余空间4byte镖师长度 + if ((position + length + 4) <= this.MAX_FILE_SIZE) { + mappedByteBuffer.position((int) position); + mappedByteBuffer.putInt(length); + mappedByteBuffer.put(data); + this.writePosition.addAndGet(length + 4); + this.lastFlushFilePosition.addAndGet(length + 4); + // 更新文件前16字节writePostion + mappedByteBuffer.position(8); + mappedByteBuffer.putLong(position + length + 4); + mappedByteBuffer.position(8); + // logger.error(" 写入 mappedByteBuffer.position(8)=="+mappedByteBuffer.getLong()); + return true; + } + return false; + } + + /** + * 设置文件的首写地址 + * + * @param pos + */ + public void setWritePositionToFile(long pos) { + mappedByteBuffer.position(8); + mappedByteBuffer.putLong(pos); + } + + public boolean saveData(long data) { + long position = this.writePosition.get(); + logger.debug("save long this.writePosition.get() " + position); + // 表示有空余空间 + if ((position + 8) <= this.MAX_FILE_SIZE) { + mappedByteBuffer.position((int) position); + mappedByteBuffer.putLong(data); + this.writePosition.addAndGet(8); + this.lastFlushFilePosition.addAndGet(8); + // 更新文件前16字节writePostion + mappedByteBuffer.position(8); + mappedByteBuffer.putLong(this.writePosition.get()); + return true; + } + return false; + } + + public boolean saveData(int data) { + long position = this.writePosition.get(); + logger.debug("save int this.writePosition.get() " + position); + // 表示有空余空间 + if ((position + 4) <= this.MAX_FILE_SIZE) { + mappedByteBuffer.position((int) position); + mappedByteBuffer.putInt(data); + this.writePosition.addAndGet(4); + // 更新文件前16字节writePostion + mappedByteBuffer.position(8); + mappedByteBuffer.putLong(this.writePosition.get()); + return true; + } + return false; + } + + /** + * 进行刷新，将数据写入磁盘 + * + * @return + */ + public long flush() { + this.mappedByteBuffer.force(); + lastFlushFilePosition.set(this.writePosition.get()); + this.lastFlushFilePosition = this.writePosition; + return this.lastFlushFilePosition.get(); + } + + /** + * 获取指定的位置的文件字节数据 暂时不用 + * + * @param start + * @param end + * @return + */ + public byte[] getData(long start, long end) { + if (start < 0 || start > this.writePosition.get()) { + return null; + } + if (end > this.writePosition.get()) { + end = this.writePosition.get(); + } + int size = (int) (end - start); + byte[] temp = new byte[size]; + this.mappedByteBuffer.position((int) start); + this.mappedByteBuffer.get(temp); + return temp; + } + + /** + * 获取从读位置到写位置的所有数据作为一个数组 恢复订阅关系使用 + * + * @return + */ + public byte[] getData() { + int size = (int) (this.writePosition.get() - this.readPosition.get()); + logger.debug(this.writePosition.get() + " , " + + this.readPosition.get()); + byte[] temp = new byte[size]; + this.mappedByteBuffer.position((int) this.readPosition.get()); + this.mappedByteBuffer.get(temp); + return temp; + } + + public AtomicLong getWritePosition() { + return writePosition; + } + + public void setWritePosition(AtomicLong writePosition) { + this.writePosition = writePosition; + } + + public AtomicLong getReadPosition() { + return readPosition; + } + + public void setReadPosition(AtomicLong readPosition) { + this.readPosition = readPosition; + } + + public AtomicLong getLastFlushFilePosition() { + return lastFlushFilePosition; + } + + public void setLastFlushFilePosition(AtomicLong lastFlushFilePosition) { + this.lastFlushFilePosition = lastFlushFilePosition; + } + + public long getLastFlushTime() { + return lastFlushTime; + } + + public void setLastFlushTime(long lastFlushTime) { + this.lastFlushTime = lastFlushTime; + } + + public String getFilePathAndName() { + return filePathAndName; + } + + public void setFilePathAndName(String filePathAndName) { + this.filePathAndName = filePathAndName; + } + + public RandomAccessFile getRaf() { + return raf; + } + + public void setRaf(RandomAccessFile raf) { + this.raf = raf; + } + + public MappedByteBuffer getMappedByteBuffer() { + return mappedByteBuffer; + } + + public void setMappedByteBuffer(MappedByteBuffer mappedByteBuffer) { + this.mappedByteBuffer = mappedByteBuffer; + } + + public FileChannel getFileChannel() { + return fileChannel; + } + + public void setFileChannel(FileChannel fileChannel) { + this.fileChannel = fileChannel; + } + + public static long getMaxFileSize() { + return MAX_FILE_SIZE; + } + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/store/Message.java b/src/main/java/com/alibaba/middleware/race/mom/store/Message.java new file mode 100644 index 0000000..5dc7b86 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/store/Message.java @@ -0,0 +1,129 @@ +package com.alibaba.middleware.race.mom.store; + + +import java.io.Serializable; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +public class Message implements Serializable{ + /** + * + */ + private static final long serialVersionUID = 5295808332504208830L; + private String topic; + private byte[] body; + //全局唯一的消息id，不同消息不能重复 + private String msgId; + private int offset; + private long bornTime; + + private Map properties = new HashMap(); + + public int getOffset() { + return offset; + } + public void setOffset(int offset) { + this.offset = offset; + } + public void setTopic(String topic) { + this.topic = topic; + } + public String getMsgId() { + return msgId; + } + public void setMsgId(String msgId) { + this.msgId = msgId; + } + public String getTopic() { + return topic; + } + + public void setBody(byte[] body) { + this.body = body; + } + + public byte[] getBody() { + return body; + } + + public String getProperty(String key) { + return properties.get(key); + } + /** + * 设置消息属性 + * @param key + * @param value + */ + public void setProperty(String key, String value) { + properties.put(key, value); + } + /** + * 删除消息属性 + * @param key + */ + public void removeProperty(String key) { + properties.remove(key); + } + public long getBornTime() { + return bornTime; + } + public void setBornTime(long bornTime) { + this.bornTime = bornTime; + } + public Map getProperties() { + return properties; + } + public void setProperties(Map properties) { + this.properties = properties; + } + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + Arrays.hashCode(body); + result = prime * result + (int) (bornTime ^ (bornTime >>> 32)); + result = prime * result + ((msgId == null) ? 0 : msgId.hashCode()); + result = prime * result + ((properties == null) ? 0 : properties.hashCode()); + result = prime * result + ((topic == null) ? 0 : topic.hashCode()); + return result; + } + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Message other = (Message) obj; + if (!Arrays.equals(body, other.body)) + return false; + if (bornTime != other.bornTime) + return false; + if (msgId == null) { + if (other.msgId != null) + return false; + } else if (!msgId.equals(other.msgId)) + return false; + if (properties == null) { + if (other.properties != null) + return false; + } else if (!properties.equals(other.properties)) + return false; + if (topic == null) { + if (other.topic != null) + return false; + } else if (!topic.equals(other.topic)) + return false; + return true; + } + @Override + public String toString() { + return "Message [topic=" + topic + ", body=" + Arrays.toString(body) + ", msgId=" + msgId + ", bornTime=" + + bornTime + ", properties=" + properties + "]"; + } + + + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/store/MsgFileUtil.java b/src/main/java/com/alibaba/middleware/race/mom/store/MsgFileUtil.java new file mode 100644 index 0000000..a54032f --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/store/MsgFileUtil.java @@ -0,0 +1,161 @@ +package com.alibaba.middleware.race.mom.store; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.LinkedList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.fastjson.JSON; +import com.alibaba.middleware.race.mom.Message; +/** + *测试用的代码，是这样提供功能吗？ + *没有考虑内存映射，优化，锁，并发，只是功能， + *大家看一下，这些接口提供了你们想的信息吗？ + * 不明白，offset ，我暂时理解为在index中的实际位置，根据这个位置可以读一个long值代表datafile中的实际数据的地址。 + */ +public class MsgFileUtil { + Logger logger = LoggerFactory.getLogger(MsgFileUtil.class); + static String fileRoot = System.getProperty("user.home"); + static String fileSpt = System.getProperty("file.separator"); + + private RandomAccessFile getFile(String topic, String groupId, String queueId, String fileName) throws IOException { + String fileStr; + if (queueId == null || queueId.equals("") || queueId.isEmpty()) { + // fileStr = fileRoot + fileSpt + "store" + fileSpt + "normal" + + // fileSpt + topic + fileSpt + queueId + fileSpt +fileName; + fileStr = fileRoot + "/store/normal/" + topic + "/" + queueId + "/" + fileName; + } else { + // fileStr = fileRoot + fileSpt + "store" + fileSpt + "retry" + + // fileSpt + groupId + fileSpt + fileName; + fileStr = fileRoot + "/store/retry/" + groupId + "/" + fileName; + } + File f = new File(fileStr); + if (!f.exists()) { + if (!f.getParentFile().exists()) { + f.getParentFile().mkdirs(); + } + f.createNewFile(); + } + return new RandomAccessFile(f, "rw"); + } + + /* + * 给定 索引文件的地址offset，返回对应data文件的msg + * + */ + public List getMsg(String topic, String queueId, String groupId, Long offset) throws IOException { + if (offset < 0) { + return null; + } + return getMsg(topic, queueId, groupId, offset, offset); + } + + /* + * 根据请求参数传回List + * + * + */ + public List getMsg(String topic, String queueId, String groupId, Long offset, Long maxOffset) + throws IOException { + List resultMsg = new LinkedList(); + RandomAccessFile indexFile, dataFile; + // 根据是否有queueId判定文件位置 + if (queueId == null || queueId.equals("") || queueId.isEmpty()) { + indexFile = getFile(topic, null, queueId, "index.file"); + dataFile = getFile(topic, null, queueId, "data.file"); + } else { + indexFile = getFile(null, groupId, null, "index.file"); + dataFile = getFile(null, groupId, null, "data.file"); + } + + // 找到index文件的offset读出真正的data文件的offset + + while (offset <= maxOffset && offset < indexFile.length()) { + indexFile.seek(offset); + long dataOffset = indexFile.readLong(); + int dataSize = indexFile.readInt(); + // 根据dataoffset读出 msg + dataFile.seek(dataOffset); + byte[] msgBody = new byte[dataSize]; + dataFile.read(msgBody); + // 读出内容将index文件的type类型设置为已经读“1” + indexFile.seek(offset + 12); + indexFile.writeInt(1); + Message msgTemp = JSON.parseObject(msgBody, Message.class); + resultMsg.add(msgTemp); + offset += 16; + } + return resultMsg; + + } + + /* + * 返回一个队列中所有消息 + */ + public List getMsg(String topic, String queueId, String groupId) throws IOException { + long offset = 0; + long maxOffset = Long.MAX_VALUE; + return getMsg(topic, queueId, groupId, offset, maxOffset); + } + + /* + * 给定 indexfile unit { offen(long),length(int),type(int)} + */ + public boolean setMsg(String topic, String queueId, String groupId, Message msg) throws Exception { + RandomAccessFile indexFile, dataFile; + if (queueId == null || queueId.equals("") || queueId.isEmpty()) { + indexFile = getFile(topic, null, queueId, "index.file"); + dataFile = getFile(topic, null, queueId, "data.file"); + } else { + indexFile = getFile(null, groupId, null, "index.file"); + dataFile = getFile(null, groupId, null, "data.file"); + } + byte[] msgByte = JSON.toJSONBytes(msg); + long dataPos = dataFile.length(); + long indexPos = indexFile.length(); + //System.out.println("indexpos,datapos=" + indexPos + "," + dataPos); + // 写入index + indexFile.seek(indexPos); + indexFile.writeLong(dataPos); + indexFile.writeInt(msgByte.length); + indexFile.writeInt(0); + // 写入data + dataFile.seek(dataPos); + dataFile.write(msgByte); + //System.out.println("indexpos,datapos=" + indexFile.length() + "," + dataFile.length()); + /* + * //恢复数据 test dataPos=dataFile.length(); indexPos=indexFile.length(); + * for(long i=0;i msgList); + + // 取消息每个消息的字节码存为list返回 + public List readByteNormal(String topicID, String queueId, + int offset, int MaxOffset); + + // 存储重试消息, 成功true,失败false + public boolean writeByteRetry(String groupId, List msgList); + + // 取重试消息每个消息的字节码存为list返回 + public ArrayList readByteRetry(String groupId, int offset, + int MaxOffset); + + // 组写 为数据一组为单位，在index 与data 文件分开写 + boolean writeByteNormalByOnce(String topicId, String queueId, + List msgList); + + // 组写 为数据一组为单位，在index 与data 文件分开写 + boolean writeByteRetryByOnce(String groupId, List msgList); + + // 组读 为数据一组为单位，在index 与data 文件分开读 + List readByteNormalByOnce(String topicId, String queueId, + int offset, int MaxOffset); + + // 组读 为数据一组为单位，在index 与data 文件分开读 + List readByteRetryByOnce(String queueId, int offset, int MaxOffset); + + // 落盘-临时文件缓冲 + /** + * 我传你的文件，你按顺序加入你的落盘逻辑，文件里的格式( 消息大小(int)+消息实体(body)) + * 你需要一条一条读，然后读完一个文件刷一次就行。刷完就删除哈。 记住，可能你还没读完的时候 + * 我又调用这个api传你另外一个文件，所以你需要加入一个链表。 按顺序用完就删，不断的取。 + * + * @param filename + */ + public void writeMappedFileTemp(String filename); + + // test use + public void testprintlong(); + + public void loadAndRecover(); +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/store/MsgStoreImp_MappedBuffer.java b/src/main/java/com/alibaba/middleware/race/mom/store/MsgStoreImp_MappedBuffer.java new file mode 100644 index 0000000..731a268 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/store/MsgStoreImp_MappedBuffer.java @@ -0,0 +1,399 @@ +//package com.alibaba.middleware.race.mom.store; +// +//import java.io.File; +//import java.io.FileDescriptor; +//import java.io.IOException; +//import java.io.RandomAccessFile; +//import java.nio.MappedByteBuffer; +//import java.nio.channels.FileChannel; +//import java.nio.channels.FileChannel.MapMode; +//import java.util.ArrayList; +//import java.util.List; +//import java.util.concurrent.ConcurrentHashMap; +//import java.util.concurrent.CountDownLatch; +//import java.util.concurrent.ExecutorService; +//import java.util.concurrent.Executors; +//import java.util.concurrent.atomic.AtomicLong; +// +//import org.slf4j.Logger; +//import org.slf4j.LoggerFactory; +// +///* +// * baseDir/Normal/Topic/Group/Consumer/Queue/(index.file,data.file) +// * baseDir/Retry/Group/(index.file,data.file) +// * 题目没有consumerID就默认生成和groupID一样的 +// * */ +// +//public class MsgStoreImp_MappedBuffer implements MsgStore { +// private static Logger logger = LoggerFactory.getLogger(MsgStoreImp_MappedBuffer.class); +// public static final int OS_PAGE_SIZE = 1024 * 4; +// // 当前JVM中映射的虚拟内存总大小 +// /*private static final AtomicLong TotalMapedVitualMemory = new AtomicLong(0); +// // 当前JVM中mmap句柄数量 +// private static final AtomicInteger TotalMapedFiles = new AtomicInteger(0);*/ +// // 文件大小 +// private final static long MAX_FILE_SIZE = 1024 * 1024 * 1024; +// // 文件写入位置 +// private AtomicLong indexWritePosition = new AtomicLong(0); +// private AtomicLong dataWritePosition = new AtomicLong(0); +// // 最后一次刷数据的时间 +// private long lastFlushTime = System.currentTimeMillis(); +// // 上一次刷的文件位置 +// private AtomicLong lastFlushFilePosition = new AtomicLong(0); +// // 最大的脏数据量，系统必须触发一次强制刷 +// private long MAX_INDEX_BYTE=16; +// private long MAX_FLUSH_DATA_SIZE = 16 *1; // indexsize=16byte 时间和字节共同 +// // 最大的时间间隔，系统必须触发一次强制刷 +// private long MAX_FLUSH_TIME_GAP = 25; //30ms 因为一次记录写入时间基本在30ms 误差一两条 +// RandomAccessFile indexFile, dataFile; +// FileDescriptor indexFileDescriptor; +// FileDescriptor dataFileDescriptor; +// ConcurrentHashMap fileHashMap = new ConcurrentHashMap(); +// ConcurrentHashMap mappedByteBufferHashMap = new ConcurrentHashMap(); +// MappedByteBuffer dataMappedByteBuffer, indexMappedByteBuffer; +// +// @Override +// public void init() { +// // TODO Auto-generated method stub +// +// } +// +// @Override +// public void close() { +// // TODO Auto-generated method stub +// try { +// +// if (indexFile != null) { +// indexFile.close(); +// } +// if (dataFile != null) { +// dataFile.close(); +// } +// +// } catch (IOException e) { +// // TODO Auto-generated catch block +// e.printStackTrace(); +// } finally { +// +// } +// } +// +// public synchronized void getPositionForIndexAndData() { +// try { +// // 开始定位到第一条没消费记录 +// if (this.indexWritePosition.get() == 0) { +// long i = 0; +// long length = indexFile.length(); +// // logger.debug("indexfile.length= " + length); +// while (i < length) { +// // 定位到未读的标示8+4+4 +// indexMappedByteBuffer.position((int) (i + 12)); +// int temp = indexMappedByteBuffer.getInt(); +// if (temp == -1) { +// i += 16; +// continue; +// } +// break; +// } +// +// if (i < 16) { +// this.dataWritePosition.set(0); +// this.indexWritePosition.set(0); +// } else { +// indexMappedByteBuffer.position((int) (i - 16)); +// this.dataWritePosition.set(indexMappedByteBuffer.getLong() + indexMappedByteBuffer.getInt()); +// this.indexWritePosition.set(i); +// } +// +// logger.debug("this.idnexwirtepostion== ,this.dataWritePostion== " + this.indexWritePosition.get() + ", " +// + this.dataWritePosition.get()); +// } +// +// } catch (IOException e) { +// // TODO Auto-generated catch block +// e.printStackTrace(); +// } +// } +// +// public synchronized void getMappedByteBuffer(String topicId, String groupId, String consumerId, String queueId) { +// if (topicId == null || topicId.isEmpty()) { +// indexFile = getFile(null, groupId, null, null, StoreConfig.indexFile); +// dataFile = getFile(null, groupId, null, null, StoreConfig.dataFile); +// } else { +// if (consumerId == null || consumerId.isEmpty()) { +// consumerId = groupId; +// } +// indexFile = getFile(topicId, groupId, consumerId, queueId, StoreConfig.indexFile); +// dataFile = getFile(topicId, groupId, consumerId, queueId, StoreConfig.dataFile); +// } +// +// indexMappedByteBuffer = MapedFile(indexFile, 0, MAX_FILE_SIZE); +// dataMappedByteBuffer = MapedFile(dataFile, 0, MAX_FILE_SIZE); +// } +// +// public boolean saveMessage(String topicId, String groupId, String consumerId, String queueId, +// List msgList) { +// +// getMappedByteBuffer(topicId, groupId, consumerId, queueId); +// getPositionForIndexAndData(); +// byte[] tempMsg = null; +// for (int i = 0; i < msgList.size(); i++) { +// tempMsg = msgList.get(i); +// // 写入 index +// if (i == 0) { +// indexMappedByteBuffer.position((int) this.indexWritePosition.get()); +// } +// +// indexMappedByteBuffer.putLong(this.dataWritePosition.get()); +// indexMappedByteBuffer.putInt(tempMsg.length); +// indexMappedByteBuffer.putInt(-1); +// this.indexWritePosition.addAndGet(16); +// // this.indexWritePosition.set(indexMappedByteBuffer.position()); +// // 写入data +// if (i == 0) { +// dataMappedByteBuffer.position((int) this.dataWritePosition.get()); +// } +// +// dataMappedByteBuffer.put(tempMsg); +// // this.dataWritePosition += msg.length; +// this.dataWritePosition.addAndGet(tempMsg.length); +// // this.dataWritePosition.set(dataMappedByteBuffer.position()); +// } +// if (this.dataWritePosition.get() >= MAX_FILE_SIZE || this.indexWritePosition.get() >= MAX_FILE_SIZE) { +// // this.dataWritePosition -= msg.length; +// // this.indexWritePosition -= 16; +// this.dataWritePosition.addAndGet(-(tempMsg.length)); +// this.indexWritePosition.addAndGet(-16); +// logger.debug("文件内存空间不够！"); +// flush(); +// return false; +// } +// +// long st1=System.currentTimeMillis(); +// flush(); +// +// return true; +// +// } +// +// public boolean saveMessage(String topicId, String groupId, String consumerId, String queueId, byte[] msg) { +// +// getMappedByteBuffer(topicId, groupId, consumerId, queueId); +// getPositionForIndexAndData(); +// // 写入 index +// indexMappedByteBuffer.position((int) this.indexWritePosition.get()); +// indexMappedByteBuffer.putLong(this.dataWritePosition.get()); +// indexMappedByteBuffer.putInt(msg.length); +// indexMappedByteBuffer.putInt(-1); +// this.indexWritePosition.addAndGet(16); +// // 写入data +// dataMappedByteBuffer.position((int) this.dataWritePosition.get()); +// dataMappedByteBuffer.put(msg); +// // this.dataWritePosition += msg.length; +// this.dataWritePosition.addAndGet(msg.length); +// if (this.dataWritePosition.get() >= MAX_FILE_SIZE || this.indexWritePosition.get() >= MAX_FILE_SIZE) { +// // this.dataWritePosition -= msg.length; +// // this.indexWritePosition -= 16; +// this.dataWritePosition.addAndGet(-msg.length); +// this.indexWritePosition.addAndGet(-16); +// logger.debug("文件内存空间不够！"); +// flush(); +// return false; +// } +// flush(); +// return true; +// +// } +// CountDownLatch countDownLatch; +// ExecutorService flushPool = Executors.newFixedThreadPool(2); +// public synchronized void flush() { +// countDownLatch=new CountDownLatch(2); +// flushPool.execute(new Thread(){ +// @Override +// public void run() { +// // TODO Auto-generated method stub +// // long t1=System.currentTimeMillis(); +// try{ +// MsgStoreImp_MappedBuffer.this.indexMappedByteBuffer.force(); +// }finally{ +// countDownLatch.countDown(); +// } +// // long t2=System.currentTimeMillis(); +// // System.out.println(" force index t2-t1=="+(t2-t1)); +// } +// }); +// flushPool.execute(new Thread(){ +// @Override +// public void run() { +// // TODO Auto-generated method stub +// //long t1=System.currentTimeMillis(); +// try{ +// MsgStoreImp_MappedBuffer.this.dataMappedByteBuffer.force(); +// }finally{ +// countDownLatch.countDown(); +// +// } +// // long t2=System.currentTimeMillis(); +// //System.out.println(" force data t2-t1=="+(t2-t1)); +// } +// }); +// try { +// countDownLatch.await(); +// } catch (InterruptedException e) { +// // TODO Auto-generated catch block +// e.printStackTrace(); +// } +// // flushPool.shutdown(); +// this.lastFlushTime = System.currentTimeMillis(); +// this.lastFlushFilePosition = this.indexWritePosition; +// } +// +// @Override +// public boolean writeByteNormal(String topicId, String groupId, String consumerId, String queueId, byte[] msg) { +// // TODO Auto-generated method stub +// +// return saveMessage(topicId, groupId, consumerId, queueId, msg); +// +// } +// +// public boolean writeByteNormal(String topicId, String groupId, String consumerId, String queueId, +// List msgList) { +// // TODO Auto-generated method stub +// return saveMessage(topicId, groupId, consumerId, queueId, msgList); +// } +// +// public synchronized MappedByteBuffer MapedFile(RandomAccessFile raf, long startPos, long fileSize) { +// MappedByteBuffer mappedByteBuffer = mappedByteBufferHashMap.get(raf); +// if (mappedByteBuffer != null) { +// return mappedByteBuffer; +// } +// try { +// FileChannel fileChannel = raf.getChannel(); +// mappedByteBuffer = fileChannel.map(MapMode.READ_WRITE, startPos, fileSize); +// /*TotalMapedVitualMemory.addAndGet(fileSize); +// TotalMapedFiles.incrementAndGet();*/ +// } catch (Exception e) { +// logger.debug("内存映射沒有建立"); +// e.printStackTrace(); +// } +// mappedByteBufferHashMap.put(raf, mappedByteBuffer); +// return mappedByteBuffer; +// } +// +// private RandomAccessFile getFile(String topicId, String groupId, String consumerId, String queueId, +// String fileName) { +// String file; +// if (topicId == null || topicId.isEmpty()) { +// file = StoreConfig.baseDir + StoreConfig.retryDir + groupId + "/" + fileName; +// } else { +// file = StoreConfig.baseDir + StoreConfig.normalDir + topicId + "/" + groupId + "/" + consumerId + "/" +// + queueId + "/" + fileName; +// } +// // 查找hashmap +// RandomAccessFile raf = fileHashMap.get(file); +// if (raf != null) { +// return raf; +// } +// File f = new File(file); +// try { +// if (!f.exists()) { +// if (!f.getParentFile().exists()) { +// f.getParentFile().mkdirs(); +// } +// f.createNewFile(); +// } +// raf = new RandomAccessFile(f, "rw"); +// } catch (Exception e) { +// // TODO Auto-generated catch block +// e.printStackTrace(); +// } +// fileHashMap.put(file, raf); +// return raf; +// } +// +// /* +// * 根据请求参数传回List +// * +// * +// */ +// public ArrayList getMessage(String topicId, String groupId, String consumerId, String queueId, int offset, +// int maxOffset) { +// ArrayList resultList = new ArrayList(); +// if (topicId == null || topicId.isEmpty()) { +// indexFile = getFile(null, groupId, null, null, StoreConfig.indexFile); +// dataFile = getFile(null, groupId, null, null, StoreConfig.dataFile); +// } else { +// if (consumerId == null || consumerId.isEmpty()) { +// consumerId = groupId; +// } +// indexFile = getFile(topicId, groupId, consumerId, queueId, StoreConfig.indexFile); +// dataFile = getFile(topicId, groupId, consumerId, queueId, StoreConfig.dataFile); +// } +// +// long len = 0; +// try { +// len = indexFile.length() / 16; +// } catch (IOException e1) { +// // TODO Auto-generated catch block +// e1.printStackTrace(); +// } +// +// while (offset <= maxOffset && offset < len) { +// // long+int+int =16 +// try { +// int indexoffset = offset * 16; +// indexFile.seek(indexoffset); +// long dataOffset = indexFile.readLong(); +// int dataSize = indexFile.readInt(); +// // 根据dataoffset读出 msg +// dataFile.seek(dataOffset); +// byte[] msgBody = new byte[dataSize]; +// dataFile.read(msgBody); +// // 读出内容将index文件的type类型设置为已经读“1” +// indexFile.seek(indexoffset + 12); +// indexFile.writeInt(1); +// resultList.add(msgBody); +// offset += 1; +// } catch (IOException e) { +// // TODO Auto-generated catch block +// e.printStackTrace(); +// return resultList; +// } +// } +// +// return resultList; +// +// } +// +// @Override +// public ArrayList readByteNormal(String topicId, String groupId, String consumerId, String queueId, +// int offset, int maxOffset) { +// // TODO Auto-generated method stub +// return getMessage(topicId, groupId, consumerId, queueId, offset, maxOffset); +// +// } +// +// @Override +// public boolean writeByteRetry(String groupId, byte[] msg) { +// // TODO Auto-generated method stub +// return saveMessage(null, groupId, null, null, msg); +// } +// +// @Override +// public ArrayList readByteRetry(String groupId, int offset, int maxOffset) { +// +// return getMessage(null, groupId, null, null, offset, maxOffset); +// +// } +// +// public static void main(String[] args) { +// +// } +// +// @Override +// public List readByteByOffsets(String topicID, String group, +// String consumer, String queueID, Integer[] offsets) { +// // TODO Auto-generated method stub +// return null; +// } +//} diff --git a/src/main/java/com/alibaba/middleware/race/mom/store/MsgStoreImp_MappedBuffer2.java b/src/main/java/com/alibaba/middleware/race/mom/store/MsgStoreImp_MappedBuffer2.java new file mode 100644 index 0000000..499817d --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/store/MsgStoreImp_MappedBuffer2.java @@ -0,0 +1,239 @@ +package com.alibaba.middleware.race.mom.store; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.MappedByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/* + * baseDir/Normal/Topic/Group/Consumer/Queue/(index.file,data.file) + * baseDir/Retry/Group/(index.file,data.file) + * 题目没有consumerID就默认生成和groupID一样的 + * */ + +public class MsgStoreImp_MappedBuffer2 implements MsgStore { + private static Logger logger = LoggerFactory + .getLogger(MsgStoreImp_MappedBuffer2.class); + private static ConcurrentHashMap fileStoreManagerMap = new ConcurrentHashMap(); + + @Override + public void init() { + // TODO Auto-generated method stub + + } + + @Override + public void close() { + // TODO Auto-generated method stub + fileStoreManagerMap.clear(); + } + + @Override + public boolean writeByteNormal(String topicId, String queueId, + List msgList) { + // 存储文件路径 + String filePath = StoreConfig.baseDir + StoreConfig.normalDir + topicId + + "/" + queueId; + // 对应的文件管理器 + FileStoreManager fileStoreManager = fileStoreManagerMap.get(filePath); + if (fileStoreManager == null) { + fileStoreManager = new FileStoreManager(filePath); + fileStoreManagerMap.put(filePath, fileStoreManager); + logger.debug("建立filestoremanager= " + filePath); + } + return fileStoreManager.saveMessage(msgList); + + } + + @Override + public void testprintlong() { + System.out.println("FileStoreManager.savemessage" + + FileStoreManager.savemessage); + System.err.println("FileStoreManager.savemessagebyonce" + + FileStoreManager.savemessagebyonce); + } + + @Override + public boolean writeByteNormalByOnce(String topicId, String queueId, + List msgList) { + // 存储文件路径 + String filePath = StoreConfig.baseDir + StoreConfig.normalDir + topicId + + "/" + queueId; + // 对应的文件管理器 + FileStoreManager fileStoreManager = fileStoreManagerMap.get(filePath); + if (fileStoreManager == null) { + fileStoreManager = new FileStoreManager(filePath); + fileStoreManagerMap.put(filePath, fileStoreManager); + logger.debug("建立filestoremanager= " + filePath); + } + return fileStoreManager.saveMessageByOnce(msgList); + + } + + @Override + public boolean writeByteRetry(String groupId, List msgList) { + // TODO Auto-generated method stub + // 存储文件路径 + String filePath = StoreConfig.baseDir + StoreConfig.retryDir + groupId; + // 对应的文件管理器 + FileStoreManager fileStoreManager = fileStoreManagerMap.get(filePath); + if (fileStoreManager == null) { + fileStoreManager = new FileStoreManager(filePath); + fileStoreManagerMap.put(filePath, fileStoreManager); + } + return fileStoreManager.saveMessage(msgList); + } + + @Override + public boolean writeByteRetryByOnce(String groupId, List msgList) { + // TODO Auto-generated method stub + // 存储文件路径 + String filePath = StoreConfig.baseDir + StoreConfig.retryDir + groupId; + // 对应的文件管理器 + FileStoreManager fileStoreManager = fileStoreManagerMap.get(filePath); + if (fileStoreManager == null) { + fileStoreManager = new FileStoreManager(filePath); + fileStoreManagerMap.put(filePath, fileStoreManager); + } + return fileStoreManager.saveMessageByOnce(msgList); + } + + @Override + public ArrayList readByteRetry(String groupId, int offset, + int MaxOffset) { + // 存储文件路径 + String filePath = StoreConfig.baseDir + StoreConfig.retryDir + groupId; + // 对应的文件管理器 + FileStoreManager fileStoreManager = fileStoreManagerMap.get(filePath); + if (fileStoreManager == null) { + fileStoreManager = new FileStoreManager(filePath); + fileStoreManagerMap.put(filePath, fileStoreManager); + } + return (ArrayList) fileStoreManager.getMessage(offset, MaxOffset); + + } + + @Override + public List readByteNormal(String topicId, String queueId, + int offset, int MaxOffset) { + // 存储文件路径 + String filePath = StoreConfig.baseDir + StoreConfig.normalDir + topicId + + "/" + queueId; + // 对应的文件管理器 + FileStoreManager fileStoreManager = fileStoreManagerMap.get(filePath); + if (fileStoreManager == null) { + fileStoreManager = new FileStoreManager(filePath); + fileStoreManagerMap.put(filePath, fileStoreManager); + logger.debug("读数据建立filestoremanage"); + } + logger.debug("读数据的起始地址为" + offset + " " + MaxOffset); + return (ArrayList) fileStoreManager.getMessage(offset, MaxOffset); + } + + @Override + public List readByteNormalByOnce(String topicId, String queueId, + int offset, int MaxOffset) { + // 存储文件路径 + String filePath = StoreConfig.baseDir + StoreConfig.normalDir + topicId + + "/" + queueId; + // 对应的文件管理器 + FileStoreManager fileStoreManager = fileStoreManagerMap.get(filePath); + if (fileStoreManager == null) { + fileStoreManager = new FileStoreManager(filePath); + fileStoreManagerMap.put(filePath, fileStoreManager); + logger.debug("读数据建立filestoremanage"); + } + logger.debug("读数据的起始地址为" + offset + " " + MaxOffset); + return (ArrayList) fileStoreManager.getMessageByOnce(offset, MaxOffset); + } + + @Override + public List readByteRetryByOnce(String queueId, int offset, + int MaxOffset) { + // 存储文件路径 + String filePath = StoreConfig.baseDir + StoreConfig.normalDir + "/" + + queueId; + // 对应的文件管理器 + FileStoreManager fileStoreManager = fileStoreManagerMap.get(filePath); + if (fileStoreManager == null) { + fileStoreManager = new FileStoreManager(filePath); + fileStoreManagerMap.put(filePath, fileStoreManager); + logger.debug("读数据建立filestoremanage"); + } + logger.debug("读数据的起始地址为" + offset + " " + MaxOffset); + return (ArrayList) fileStoreManager.getMessageByOnce(offset, MaxOffset); + + } + + @Override + public void writeMappedFileTemp(String filename) { + // TODO Auto-generated method stub + + } + + @Override + public void loadAndRecover() { + // 恢复正常消息队列 + String filePath = StoreConfig.baseDir + StoreConfig.normalDir; + ArrayList rsPath = this.scanFilePathForNormal(filePath); + if (rsPath == null) + return; + for (String[] path : rsPath) { + // 存储文件路径 + logger.error(filePath.toString()); + filePath = StoreConfig.baseDir + StoreConfig.normalDir+path[0] + "/" + path[1]; + logger.error(filePath.toString()); + // 对应的文件管理器 + FileStoreManager fileStoreManager = fileStoreManagerMap + .get(filePath); + if (fileStoreManager == null) { + fileStoreManager = new FileStoreManager(filePath, "testrecover"); + fileStoreManagerMap.put(filePath, fileStoreManager); + } + } + // 恢复订阅关系 + filePath = StoreConfig.baseDir + StoreConfig.subscribeDir; + logger.error("subFfilepath="+filePath); + FileStoreManager fileStoreManager = fileStoreManagerMap.get(filePath); + if (fileStoreManager == null) { + fileStoreManager = new FileStoreManager(filePath, "testrecover"); + fileStoreManagerMap.put(filePath, fileStoreManager); + } + //恢复retry关系 + //filePath = StoreConfig.baseDir + StoreConfig.; + } + + public ArrayList scanFilePathForNormal(String filePath) { + ArrayList path = new ArrayList(); + File[] fileList = new File(filePath).listFiles(); + if (fileList == null) + return null; + for (File tempFile : fileList) { + if (tempFile.isDirectory()) { + for (File temp : tempFile.listFiles()) { + String[] tq = new String[2]; + String fs = System.getProperty("file.separator"); + System.out.println(temp.toString() + " fs= " + fs); + String t = temp.toString(); + int i = t.lastIndexOf(fs); + tq[1] = t.substring(i + 1); + int j = t.lastIndexOf(fs, i - 1); + tq[0] = t.substring(j + 1, i); + System.out.println(i + "-----" + j); + path.add(tq); + } + } + } + for (String[] t : path) { + System.out.println(t[0] + " / " + t[1]); + } + return path; + } + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/store/OffsetNum.java b/src/main/java/com/alibaba/middleware/race/mom/store/OffsetNum.java new file mode 100644 index 0000000..11868aa --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/store/OffsetNum.java @@ -0,0 +1,21 @@ +package com.alibaba.middleware.race.mom.store; + +import java.io.Serializable; + +public class OffsetNum implements Serializable { + /** + * offset 的数值 + */ + private static final long serialVersionUID = -352056806960750564L; + int i; + public int getI() { + return i; + } + public OffsetNum setI(int i) { + this.i = i; + return this; + } + public String toString(){ + return String.valueOf(i); + } +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/store/Redo.java b/src/main/java/com/alibaba/middleware/race/mom/store/Redo.java new file mode 100644 index 0000000..aa08880 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/store/Redo.java @@ -0,0 +1,133 @@ +package com.alibaba.middleware.race.mom.store; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileChannel.MapMode; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +import com.alibaba.fastjson.JSON; + +public class Redo { + public static String redoFile = System.getProperty("user.home") + + File.separator +"0"; + public MappedByteBuffer mbb; + public File file; + public RandomAccessFile raf; + public AtomicLong filename = new AtomicLong(1); + public int filesize = 1024*1024 * 1024; + private RedoBean rb=new RedoBean(); + + + public void writeLog(String msgid,String topicAndfilter,byte[] body) + { + rb.getBody().add(body); + rb.getMsgId().add(msgid); + rb.getTopicAfilter().add(topicAndfilter); + } + public void commitLog(RedoBean rb) { + byte[] smByte = JSON.toJSONBytes(rb); + if (mbb.position() + smByte.length > filesize) { + filename.getAndIncrement(); + try { + mbb = new RandomAccessFile(getNewFile(), "rw").getChannel().map( + FileChannel.MapMode.READ_WRITE, 0, filesize);; + } catch (IOException e) { + } + } + mbb.putInt(smByte.length); + mbb.put(smByte); + } + + public void flushLog() { + commitLog(rb); + mbb.force(); + rb=new RedoBean(); + } + + public Redo() { + file = new File(redoFile); + ensureDirOK(file.getParent()); + RandomAccessFile raf; + try { + raf = new RandomAccessFile(file, "rw"); + FileChannel fileChannel = raf.getChannel(); + mbb = fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, filesize); + } catch (FileNotFoundException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + } + public File getNewFile() throws FileNotFoundException, + IOException { + String newfilename = new String("" + + (Long.parseLong(file.getName()) + filesize)); + file = new File(System.getProperty("user.home") + File.separator + + "store" + File.separator + newfilename); + return file; + } + + public static void main(String args[]) throws IOException { + for(int j=0;j<100;j++) + { + Redo redo = new Redo(); + long begin=System.currentTimeMillis(); + for(int i=0;i<200;i++) + { + redo.writeLog("msg-"+i, "topic-and-filter-y", "hello mom I'm body".getBytes()); + } + redo.flushLog(); + System.out.println(System.currentTimeMillis()-begin); + } + + + } + public void ensureDirOK(final String dirName) { + if (dirName != null) { + File f = new File(dirName); + if (!f.exists()) { + f.mkdirs(); + } + } + } + class RedoBean implements Serializable{ + + /** + * 存储的实体 + */ + private static final long serialVersionUID = -5597911942382576497L; + List topicAfilter=new ArrayList<>(); + List msgId=new ArrayList<>(); + List body=new ArrayList<>(); + public List getTopicAfilter() { + return topicAfilter; + } + public void setTopicAfilter(List topicAfilter) { + this.topicAfilter = topicAfilter; + } + public List getMsgId() { + return msgId; + } + public void setMsgId(List msgId) { + this.msgId = msgId; + } + public List getBody() { + return body; + } + public void setBody(List body) { + this.body = body; + } + + } +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/store/RedoLog.java b/src/main/java/com/alibaba/middleware/race/mom/store/RedoLog.java new file mode 100644 index 0000000..79886d6 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/store/RedoLog.java @@ -0,0 +1,163 @@ +package com.alibaba.middleware.race.mom.store; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel.MapMode; +import java.util.ArrayList; + +import com.alibaba.fastjson.JSON; + +public class RedoLog { + private static String redoFile = StoreConfig.baseDir + + "oo33o" + "r2222edo"; + public MappedByteBuffer mbb; + public int fileSize = repeatNum*(msgSize+1); + public static int repeatNum = 512; + public static int msgSize = 64; + public char FLAG = 0; +//god bless + public RedoLog() { + this.mbb = this.getFileMappedByteBuffer(redoFile, 0, fileSize); + } + + // msg=head +body : + public boolean writeLog(String topic, String filter, int queueIndex, + ArrayList bodys) { +// LogBean lb=new LogBean(topic, filter, 0,bodys); +// byte[] data=JSON.toJSONBytes(lb); + for (int i = 0; i < bodys.size(); i++) { + String head = topic + FLAG + filter + FLAG + queueIndex; + byte[] headByte = head.getBytes(); + // head.size head body.szie body + ByteBuffer msgByteBuffer = ByteBuffer.allocate(msgSize); + + msgByteBuffer.putInt(headByte.length); + msgByteBuffer.put(headByte); + msgByteBuffer.putInt(bodys.get(i).length); + msgByteBuffer.put(bodys.get(i)); + boolean flag = this.writeLog(msgByteBuffer); + if (!flag) { + return false; + } + } + return true; + //this.flushLog(); + + } + + private boolean writebytes(byte[] smByte) + { + if (mbb.position() + smByte.length > fileSize) { + return false; + } + mbb.putInt(smByte.length); + mbb.put(smByte); + return true; + } + private boolean writeLog(ByteBuffer smByte) { + if (mbb.position() + smByte.limit() > fileSize) { + return false; + } + mbb.putInt(smByte.limit()); + smByte.flip(); + mbb.put(smByte); + return true; + } + + public void flushLog() { + mbb.force(); + if (mbb.position() > repeatNum * msgSize) + mbb.position(0); + } + private MappedByteBuffer getFileMappedByteBuffer(String filePathAndName, + int startPos, int fileSize) { + File f = new File(filePathAndName); + try { + if (!f.exists()) { + if (!f.getParentFile().exists()) { + f.getParentFile().mkdirs(); + } + f.createNewFile(); + } + return mbb = new RandomAccessFile(f, "rw").getChannel().map( + MapMode.READ_WRITE, startPos, fileSize); + + } catch (Exception e) { + return null; + } + } + + public static void main(String args[]) throws IOException { + RedoLog redo = new RedoLog(); + char FLAG = 0; + for (int j = 0; j < 500; j++) { + long begin = System.currentTimeMillis(); + for (int i = 0; i < 200; i++) { + /* + * String data = "topic" + i + FLAG + "fitler" + FLAG + + * "hello mom Im baby"; ByteBuffer smByte = + * ByteBuffer.allocate(redo.msgSize); + * smByte.put(data.getBytes()); redo.writeLog(smByte); + */ + ArrayList list = new ArrayList(); + list.add("1234".getBytes()); + + redo.writeLog("topic", "filter", 0, list); + } + long flushtime = System.currentTimeMillis(); + redo.flushLog(); + System.out.println("writetime:" + (flushtime - begin) + + " flushtime:" + (System.currentTimeMillis() - flushtime)); + } + + } + class LogBean + { + String topic; + String filter; + int queueid; + ArrayList list=new ArrayList<>(); + + public LogBean(String topic, String filter, int queueid, ArrayList list) { + super(); + this.topic = topic; + this.filter = filter; + this.queueid = queueid; + this.list = list; + } + public int getQueueid() { + return queueid; + } + public void setQueueid(int queueid) { + this.queueid = queueid; + } + public LogBean(String topic, String filter, ArrayList list) { + super(); + this.topic = topic; + this.filter = filter; + this.list = list; + } + public String getTopic() { + return topic; + } + public void setTopic(String topic) { + this.topic = topic; + } + public String getFilter() { + return filter; + } + public void setFilter(String filter) { + this.filter = filter; + } + public ArrayList getList() { + return list; + } + public void setList(ArrayList list) { + this.list = list; + } + + } +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/store/StoreConfig.java b/src/main/java/com/alibaba/middleware/race/mom/store/StoreConfig.java new file mode 100644 index 0000000..f74c76c --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/store/StoreConfig.java @@ -0,0 +1,28 @@ +package com.alibaba.middleware.race.mom.store; + +public class StoreConfig { + // 主目录 + public static String fileRoot = System.getProperty("user.home"); + // public static String fileSpt = System.getProperty("file.separator"); + public static String baseDir = fileRoot + "/store/"; + // 正常消息文件 + public static String normalDir = "normal/"; + // 重试消息文件 + public static String retryDir = "retry/"; + // 订阅关系 + public static String subscribeDir = "subscribe/"; + // + public static String arrayDir = "array/"; + + // 索引文件名 + /* + * 默认 offset(8byte) size(4byte) extra(4byte) + */ + public static int offsetByte = 8, sizeByte = 4, extraByte = 4; + public static String indexFile = "index.file"; + // 消息文件名 + public static String dataFile = "data.file"; + // 订阅关系文件名 + public static String subDataFile = "subData.file"; + public static String subIndexFile = "subIndex.file"; +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/store/StoreMsg.java b/src/main/java/com/alibaba/middleware/race/mom/store/StoreMsg.java new file mode 100644 index 0000000..9e7a84c --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/store/StoreMsg.java @@ -0,0 +1,21 @@ +package com.alibaba.middleware.race.mom.store; + +import java.io.Serializable; + +public class StoreMsg implements Serializable{ + /** + * + */ + private static final long serialVersionUID = -2571744737752908548L; + String id; + String topic; + int queueid; + byte[] body; + + public StoreMsg(String id, String topic, int queueid, byte[] body) { + this.id = id; + this.topic = topic; + this.queueid = queueid; + this.body = body; + } +} \ No newline at end of file diff --git a/src/main/java/com/alibaba/middleware/race/mom/store/SubscribeStore.java b/src/main/java/com/alibaba/middleware/race/mom/store/SubscribeStore.java new file mode 100644 index 0000000..0b70dd2 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/store/SubscribeStore.java @@ -0,0 +1,16 @@ +package com.alibaba.middleware.race.mom.store; + +import java.util.ArrayList; + +import com.alibaba.middleware.race.mom.broker.function.Offset; + +/* + * by ChenQi + * 订阅关系持久化 + * */ +public interface SubscribeStore { + //读出所有的订阅关系 + public ArrayList read(); + //存所有的订阅关系 + public boolean write(ArrayList subInfoList); +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/store/SubscribeStoreImp.java b/src/main/java/com/alibaba/middleware/race/mom/store/SubscribeStoreImp.java new file mode 100644 index 0000000..f59bd0b --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/store/SubscribeStoreImp.java @@ -0,0 +1,134 @@ +package com.alibaba.middleware.race.mom.store; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.ArrayList; + +import com.alibaba.fastjson.JSON; +import com.alibaba.middleware.race.mom.broker.function.Offset; + +public class SubscribeStoreImp implements SubscribeStore { + + @Override + public ArrayList read() { + // TODO Auto-generated method stub + ArrayList rs = new ArrayList(); + RandomAccessFile subIndexFile = getSubFile(StoreConfig.subIndexFile, false); + RandomAccessFile subDataFile = getSubFile(StoreConfig.subDataFile, false); + try { + long indexLength = subIndexFile.length(); + long index = 0; + long dataPos = 0; + int dataSize = 0; + byte[] subByte; + while (index < indexLength) { + subIndexFile.seek(index); + dataPos = subIndexFile.readLong(); + dataSize = subIndexFile.readInt(); + subDataFile.seek(dataPos); + subByte = new byte[dataSize]; + subDataFile.read(subByte); + rs.add(((Offset) JSON.parseObject(subByte, Offset.class))); + index += 16; + } + + } catch (Exception e) { + e.printStackTrace(); + return rs; + }finally{ + try { + subIndexFile.close(); + subDataFile.close(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + } + return rs; + } + + @Override + public boolean write(ArrayList subInfoList) { + // TODO Auto-generated method stub + RandomAccessFile subIndexFile = getSubFile(StoreConfig.subIndexFile, true); + RandomAccessFile subDataFile = getSubFile(StoreConfig.subDataFile, true); + try { + long dataPos = 0; + long indexPos = 0; + + // 每次重头写 + for (Offset sb : subInfoList) { + indexPos = subIndexFile.length(); + dataPos = subDataFile.length(); + // 写入index + subIndexFile.seek(indexPos); + subIndexFile.writeLong(dataPos); + byte[] sbByte = JSON.toJSONBytes(sb); + subIndexFile.writeInt(sbByte.length); + subIndexFile.writeInt(0); + // 写入data + subDataFile.seek(dataPos); + subDataFile.write(sbByte); + System.out.println("indexpos,datapos=" + subIndexFile.length() + "," + subDataFile.length()); + } + + } catch (Exception e) { + e.printStackTrace(); + return false; + }finally{ + try { + subIndexFile.close(); + subDataFile.close(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + } + return true; + } + + public RandomAccessFile getSubFile(String fileName, boolean isWriteMethod) { + String file; + RandomAccessFile raf = null; + file = StoreConfig.baseDir + StoreConfig.subscribeDir + fileName; + System.out.println("file=" + file); + File f = new File(file); + if (isWriteMethod) { + try { + if (!f.exists()) { + if (!f.getParentFile().exists()) { + f.getParentFile().mkdirs(); + } + f.createNewFile(); + } else { + // 删除老文件 + f.delete(); + f.createNewFile(); + } + raf = new RandomAccessFile(f, "rw"); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } else { + try { + if (!f.exists()) { + if (!f.getParentFile().exists()) { + f.getParentFile().mkdirs(); + } + f.createNewFile(); + } + raf = new RandomAccessFile(f, "rw"); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + } + return raf; + } + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/store/SubscribeStoreImp_MappedBuffer2.java b/src/main/java/com/alibaba/middleware/race/mom/store/SubscribeStoreImp_MappedBuffer2.java new file mode 100644 index 0000000..4148f2d --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/store/SubscribeStoreImp_MappedBuffer2.java @@ -0,0 +1,67 @@ +package com.alibaba.middleware.race.mom.store; + +import java.util.ArrayList; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.fastjson.JSON; +import com.alibaba.middleware.race.mom.broker.function.Offset; + +public class SubscribeStoreImp_MappedBuffer2 implements SubscribeStore { + private static Logger logger = LoggerFactory + .getLogger(SubscribeStoreImp_MappedBuffer2.class); + private static String subscribeFilePath = StoreConfig.baseDir + + StoreConfig.subscribeDir; + private static ConcurrentHashMap fileStoreManagerMap = new ConcurrentHashMap(); + + @Override + public ArrayList read() { + FileStoreManager fileStoreManager = fileStoreManagerMap + .get(subscribeFilePath); + if (fileStoreManager == null) { + fileStoreManager = new FileStoreManager(subscribeFilePath); + // 为恢复 + fileStoreManagerMap.put(subscribeFilePath, fileStoreManager); + } + return (ArrayList) fileStoreManager.getSubscribe(); + + } + + @Override + public boolean write(ArrayList subInfoList) { + // 对应的文件管理器 + FileStoreManager fileStoreManager = fileStoreManagerMap + .get(subscribeFilePath); + if (fileStoreManager == null) { + fileStoreManager = new FileStoreManager(subscribeFilePath); + // 为恢复 + fileStoreManagerMap.put(subscribeFilePath, fileStoreManager); + } + // 将写位置重置为16 + fileStoreManager.resetFilePosition(); + + ArrayList dataList = new ArrayList(subInfoList.size()); + int count = 1; + for (Offset tmp : subInfoList) { + logger.debug("保存订阅关系 i+== " + (count++) + " , " + tmp.getTopic()); + + dataList.add(JSON.toJSONBytes(tmp)); + } + + //return fileStoreManager.saveMessage(dataList); + return fileStoreManager.saveMessageByOnce(dataList); + + } + + /* + * private void deleteOldSubscribeFile() { deleteFile(subscribeFilePath + + * "/index/0", true); deleteFile(subscribeFilePath + "/data/0", true); } + * + * public void deleteFile(String file, boolean isWriteOverride) { File f = + * new File(file); if (isWriteOverride) { try { if (f.exists()) { // 删除老文件 + * f.delete(); } } catch (Exception e) { // TODO Auto-generated catch block + * logger.error(e.getMessage()); } } } + */ +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/store/Test.java b/src/main/java/com/alibaba/middleware/race/mom/store/Test.java new file mode 100644 index 0000000..2152c94 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/store/Test.java @@ -0,0 +1,231 @@ +//package com.alibaba.middleware.race.mom.store; +// +//import java.io.File; +//import java.util.ArrayList; +//import java.util.List; +//import java.util.Random; +//import java.util.concurrent.BlockingQueue; +//import java.util.concurrent.LinkedBlockingQueue; +// +//import com.alibaba.fastjson.JSON; +//import com.alibaba.middleware.race.mom.Message; +//import com.alibaba.middleware.race.mom.broker.function.Offset; +// +//public class Test { +// String topic = "topic1", group = "group1", consumer = "group1"; +// int queue = 0; +// BlockingQueue q = new LinkedBlockingQueue<>(); +// // 进度 +// int offset = 0, maxoffset = 9; +// +// static class Msg { +// String a = "a"; +// int b = 3; +// } +// +// public static void main(String args[]) { +// Test t = new Test(); +// // 内存测试 +// // t.init(); +// // t.show(); +// // 存储恢复测试 +// // t.q.clear(); +// // t.init(); +// // t.store(); +// // t.recover(); +// // t.q.clear(); +// +// // t.init(); +// /* +// * t.store2_new(); t.recover2_new(); +// */ +// t.q.clear(); +// t.init(); +// t.store2byonce(); +// t.recover2(); +// +// // t.testSub(); +// // t.testArrayList(); +// t.testFile2(); +// //t.testFile(); +// //t.scanFilePath(StoreConfig.baseDir+StoreConfig.subscribeDir); +// } +// +// public void testFile2() { +// mstore2.loadAndRecover(); +// } +// +// public void testFile() { +// // 对正常消息 +// String filePath = StoreConfig.baseDir + StoreConfig.normalDir; +// ArrayList rsPath=this.scanFilePathForNormal(filePath); +// for(String[] path:rsPath){ +// System.out.println(); +// } +// filePath=StoreConfig.baseDir + StoreConfig.subscribeDir; +// System.out.println(filePath); +// String[] rs=scanFilePath(filePath); +// } +// +// public String[] scanFilePath(String filePath) { +// ArrayList path = new ArrayList(); +// String[] fileList = new File(filePath).list(); +// if (fileList == null) +// { +// System.out.println("null"); +// return null; +// } +// +// +// for (String t : fileList) { +// System.out.println(t); +// } +// return fileList; +// } +// public ArrayList scanFilePathForNormal(String filePath) { +// ArrayList path = new ArrayList(); +// File[] fileList = new File(filePath).listFiles(); +// for (File tempFile : fileList) { +// if (tempFile.isDirectory()) { +// for (File temp : tempFile.listFiles()) { +// String[] tq = new String[2]; +// String fs = System.getProperty("file.separator"); +// System.out.println(temp.toString() + " fs= " + fs); +// String t = temp.toString(); +// int i = t.lastIndexOf(fs); +// tq[1] = t.substring(i + 1); +// int j = t.lastIndexOf(fs, i - 1); +// tq[0] = t.substring(j + 1, i); +// System.out.println(i + "-----" + j); +// path.add(tq); +// } +// } +// } +// for (String[] t : path) { +// System.out.println(t[0] + " / " + t[1]); +// } +// return path; +// } +// +// public void show() { +// // 显示 +// for (int i = 0; i < 10; i++) { +// Message m = q.poll(); +// if (i < offset) +// System.out.println("已消费:" + m.getMsgId()); +// else +// System.out.println("未消费:" + m.getMsgId()); +// } +// } +// +// MsgStore mstore2 = new MsgStoreImp_MappedBuffer2(); +// +// public void store2() { +// +// List msgList = new ArrayList(10); +// for (int i = 0; i < 10; i++) { +// Message msg = q.poll(); +// byte[] msgbyte = JSON.toJSONBytes(msg);// 这里用fjson序列化一下 +// msgList.add(msgbyte); +// } +// long startTime = System.currentTimeMillis(); +// mstore2.writeByteNormal(topic, queue + "", msgList); +// long endTime = System.currentTimeMillis(); +// System.out.println("写入msgList数据时间 endTime-StartTime=" +// + (endTime - startTime)); +// } +// +// +// public void store2byonce() { +// +// List msgList = new ArrayList(10); +// for (int i = 0; i < 10; i++) { +// Message msg = q.poll(); +// byte[] msgbyte = JSON.toJSONBytes(msg);// 这里用fjson序列化一下 +// msgList.add(msgbyte); +// } +// long startTime = System.currentTimeMillis(); +// mstore2.writeByteNormalByOnce(topic, queue + "", msgList); +// long endTime = System.currentTimeMillis(); +// System.out.println("写入msgList数据时间 endTime-StartTime=" +// + (endTime - startTime)); +// } +// +// public void recover2() { +// long startTime = System.currentTimeMillis(); +// ArrayList blist = (ArrayList) mstore2.readByteNormal( +// topic, queue + "", offset, maxoffset); +// long endTime = System.currentTimeMillis(); +// System.out.println(" 读出数据时间 endTime-StartTime=" +// + (endTime - startTime)); +// +// for (byte[] b : blist) { +// Message msg = JSON.parseObject(b, Message.class);// obj 把 byte转为对象 +// System.out.println("未消费:" + msg.getMsgId()); +// } +// } +// +// +// public void init() { +// // queue: 头-> 0 1 2 3 4(offset) 5 6 7 8 9(maxoffset) +// for (int i = 0; i < 10; i++) { +// Message msg = new Message(); +// msg.setTopic(topic); +// msg.setBornTime(System.currentTimeMillis()); +// msg.setBody(new String("测试哦" + new Random().nextLong()).getBytes()); +// System.out.println("init msg id " + msg.getMsgId()); +// q.add(msg); +// } +// } +// +// public void destory() { +// mstore2.close(); +// } +// +// public void testSub() { +// // Offset sb1 = new Offset(); +// // sb1.setTopicId("MOM-RACE-21486"); +// // sb1.setGroupId("CID_21486"); +// // sb1.setQueueId("0"); +// // sb1.setCurrentoffset(1); +// // sb1.setCacheOffset(1); +// // sb1.setMaxOffset(1); +// // Offset sb2 = new Offset(); +// // sb2.setTopicId("ccc"); +// // sb2.setGroupId("dfadfasdlfasdfas"); +// // System.out.println("size(sb1):" + (JSON.toJSONBytes(sb1)).length); +// // System.out.println("size(sb2):" + (JSON.toJSONBytes(sb2)).length); +// SubscribeStoreImp_MappedBuffer2 ssi = new SubscribeStoreImp_MappedBuffer2(); +// ArrayList ls = new ArrayList(); +// +// // ls.add(sb1); +// // ls.add(sb2); +// // ssi.write(ls); +// System.out.println("开始读出订阅关系"); +// ls = ssi.read(); +// for (Offset sb : ls) { +// System.out.println(sb.toString()); +// } +// +// } +// +// public void testArrayList() { +// OffsetNum sb1 = new OffsetNum(); +// sb1.setI(1111); +// OffsetNum sb2 = new OffsetNum(); +// sb2.setI(22222); +// System.out.println("size(sb1):" + (JSON.toJSONBytes(sb1)).length); +// System.out.println("size(sb2):" + (JSON.toJSONBytes(sb2)).length); +// ArrayStoreImp ssi = new ArrayStoreImp(); +// ArrayList ls = new ArrayList(); +// +// ls.add(sb1); +// ls.add(sb2); +// ssi.store(ls, "topicAndFilter", "0"); +// System.out.println("开始读出订阅关系"); +// ls = ssi.recover("topicAndFilter", "0"); +// for (OffsetNum sb : ls) { +// System.out.println(sb.i); +// } +// } +//} diff --git a/src/main/java/com/alibaba/middleware/race/mom/store/redotest.java b/src/main/java/com/alibaba/middleware/race/mom/store/redotest.java new file mode 100644 index 0000000..76cd03b --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/store/redotest.java @@ -0,0 +1,143 @@ +package com.alibaba.middleware.race.mom.store; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel.MapMode; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class redotest { + String redoFile = System.getProperty("user.home") + File.separator + + "store"; + public MappedByteBuffer mbb; + public File file; + public AtomicInteger fileName = new AtomicInteger(0); + public int filesize = 1024*1024; + BlockingQueue mmbStoreQue = new ArrayBlockingQueue( + 10); + + BlockingQueue mmbRecoverQue = new ArrayBlockingQueue( + 10); + + public void writeLog2(ByteBuffer smByte) { + + if (mbb.position() + smByte.limit() > filesize) { + try { + mmbRecoverQue.put(this.mbb); + this.mbb = this.mmbStoreQue.poll(3, TimeUnit.MILLISECONDS); + if (this.mbb == null) { + this.createMbb(); + } + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + // ByteBuffer by = ByteBuffer.allocate(smByte.length); + mbb.putInt(smByte.limit()); + mbb.put(smByte); + } + + public void readRecoverLog2() { + + MappedByteBuffer readMbb = null; + while (mmbRecoverQue.size() != 0) { + try { + readMbb = this.mmbRecoverQue.take(); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + // 恢复数据 + int size = readMbb.getInt(); + while (size != 0) { + byte[] msg = new byte[size]; + readMbb.get(msg); + { + // 开始写indexdata文件 + } + size = readMbb.getInt(); + } + } + } + + public void flushLog() { + mbb.force(); + } + + public redotest() { + this.createMbb(); + } + + private void createMbb() { + for (int i = 0; i < 10; i++) { + try { + mmbStoreQue.put(getFileMappedByteBuffer(redoFile + + File.separator + fileName.getAndIncrement(), 0, + filesize)); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + try { + this.mbb = mmbStoreQue.take(); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + private MappedByteBuffer getFileMappedByteBuffer(String filePathAndName, + int startPos, int fileSize) { + File f = new File(filePathAndName); + try { + if (!f.exists()) { + if (!f.getParentFile().exists()) { + f.getParentFile().mkdirs(); + } + f.createNewFile(); + } + return mbb = new RandomAccessFile(f, "rw").getChannel().map( + MapMode.READ_WRITE, startPos, fileSize); + + } catch (Exception e) { + return null; + } + } + + /* + * public MappedByteBuffer getNewFile() throws FileNotFoundException, + * IOException { String newfilename = new String("" + + * (Long.parseLong(file.getName()) + filesize)); file = new + * File(System.getProperty("user.home") + File.separator + "store" + + * File.separator + newfilename); mbb = new RandomAccessFile(file, + * "rw").getChannel().map( FileChannel.MapMode.READ_WRITE, 0, filesize); + * return mbb; } + */ + public static void main(String args[]) throws IOException { + final redotest redo = new redotest(); + char FLAG=0; + + for(int j=0;j<500;j++) + { + final long begin = System.currentTimeMillis(); + for (int i = 0; i < 200; i++) { + String data="topic"+i+FLAG+"fitler"+FLAG+"hello mom Im baby"; + ByteBuffer smByte=ByteBuffer.allocate(64); + smByte.put(data.getBytes()); + redo.writeLog2(smByte); + } + long flushtime=System.currentTimeMillis(); + redo.flushLog(); + System.out.println("writetime:"+(flushtime-begin)+" flushtime:" + + (System.currentTimeMillis() - flushtime)); + } + + } +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/util/ConsumerInfo.java b/src/main/java/com/alibaba/middleware/race/mom/util/ConsumerInfo.java new file mode 100644 index 0000000..ff77c21 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/util/ConsumerInfo.java @@ -0,0 +1,36 @@ +package com.alibaba.middleware.race.mom.util; + +import io.netty.channel.Channel; + +import java.net.SocketAddress; + +public class ConsumerInfo { + private Channel channel; + private SocketAddress consumerAddress; + + + public ConsumerInfo(Channel channel){ + this.channel=channel; + this.consumerAddress=channel.remoteAddress(); + } + public Channel channel() { + return channel; + } + + public SocketAddress consumerAddress() { + return consumerAddress; + } + @Override + public boolean equals(Object obj) { + if(this==obj){ + return true; + }else if(obj instanceof ConsumerInfo){ + ConsumerInfo another=(ConsumerInfo)obj; + return this.consumerAddress.equals(another.consumerAddress); + }else return false; + } + + + + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/util/Info.java b/src/main/java/com/alibaba/middleware/race/mom/util/Info.java new file mode 100644 index 0000000..46b74f8 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/util/Info.java @@ -0,0 +1,37 @@ +package com.alibaba.middleware.race.mom.util; + +import java.io.Serializable; + +/* + * 封装所有的请求和响应信息 + * type 请求或者响应类型 + * body 对应的数据 + * describle 类型描述信息 + * Author ilyy510 + * */ +public class Info implements Serializable{ +/** + * + */ + private static final long serialVersionUID = -2529148421210651297L; +InfoType type; +byte[] body; +public InfoType getType() { + return type; +} +public void setType(InfoType type) { + this.type = type; +} +public byte[] getBody() { + return body; +} +public void setBody(byte[] body) { + this.body = body; +} +public static long getSerialversionuid() { + return serialVersionUID; +} + + + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/util/InfoBodyConsumer.java b/src/main/java/com/alibaba/middleware/race/mom/util/InfoBodyConsumer.java new file mode 100644 index 0000000..76ac851 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/util/InfoBodyConsumer.java @@ -0,0 +1,36 @@ +package com.alibaba.middleware.race.mom.util; + +import java.io.Serializable; +import java.util.Map; + +public class InfoBodyConsumer implements Serializable{ + /** + * + */ + private static final long serialVersionUID = 4500208867228323993L; + private String groupId; + private String topic; + private Map filterMap; + public String getGroupId() { + return groupId; + } + public void setGroupId(String groupId) { + this.groupId = groupId; + } + public String getTopic() { + return topic; + } + public void setTopic(String topic) { + this.topic = topic; + } + public Map getFilterMap() { + return filterMap; + } + public void setFilterMap(Map filterMap) { + this.filterMap = filterMap; + } + public String toString(){ + return "group:"+this.groupId+" topic:"+this.topic+" filter:"+this.filterMap; + } + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/util/InfoBodyProduceList.java b/src/main/java/com/alibaba/middleware/race/mom/util/InfoBodyProduceList.java new file mode 100644 index 0000000..f0a006a --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/util/InfoBodyProduceList.java @@ -0,0 +1,28 @@ +package com.alibaba.middleware.race.mom.util; + +import java.io.Serializable; +import java.util.ArrayList; + +import com.alibaba.middleware.race.mom.broker.function.MessageInfo; + +/** + * + * @author Administrator + * 封装producer的请求信息 + */ +public class InfoBodyProduceList implements Serializable{ + + private static final long serialVersionUID = 9075042672715185764L; + ArrayList msglist=new ArrayList<>(); + public ArrayList getMsglist() { + return msglist; + } + public void setMsglist(ArrayList msglist) { + this.msglist = msglist; + } + public InfoBodyProduceList() { + + } + + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/util/InfoType.java b/src/main/java/com/alibaba/middleware/race/mom/util/InfoType.java new file mode 100644 index 0000000..2fe6c46 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/util/InfoType.java @@ -0,0 +1,12 @@ +package com.alibaba.middleware.race.mom.util; +/** + * Info消息的类型说明 + * Author ilyy510 + * */ +public enum InfoType { + PRODUCER_TO_BROKER_SENG_MESSAGE, + BROKER_TO_PRODUCER_CONFIRM_MESSAGE, + BROKER_TO_CONSUMER_PUSH_MESSAGE, + CONSUMER_TO_BROKER_CONFIRM_MESSAGE, + CONSUMER_TO_BROKER_CONNECT +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/util/InfoUtil.java b/src/main/java/com/alibaba/middleware/race/mom/util/InfoUtil.java new file mode 100644 index 0000000..406bfe3 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/util/InfoUtil.java @@ -0,0 +1,21 @@ +package com.alibaba.middleware.race.mom.util; + +public class InfoUtil { + public static byte[] arraycat(byte[] buf1,byte[] buf2) + { + byte[] bufret=null; + int len1=0; + int len2=0; + if(buf1!=null) + len1=buf1.length; + if(buf2!=null) + len2=buf2.length; + if(len1+len2>0) + bufret=new byte[len1+len2]; + if(len1>0) + System.arraycopy(buf1,0,bufret,0,len1); + if(len2>0) + System.arraycopy(buf2,0,bufret,len1,len2); + return bufret; + } +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/util/TopicAndFilter.java b/src/main/java/com/alibaba/middleware/race/mom/util/TopicAndFilter.java new file mode 100644 index 0000000..4a85e39 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/util/TopicAndFilter.java @@ -0,0 +1,87 @@ + +package com.alibaba.middleware.race.mom.util; + +import java.io.Serializable; +import java.util.Map; +/** + * topic and filter的封装,一种TopicAndFilter代表一种消息,来自同一生产者集群组 + * 订阅者集群组 订阅某一TopicAndFilter(注意,订阅者filter为null,则订阅topic下所有filter消息) + * 使用注意:构建订阅者TopicAndFilter须根据订阅者信息; 构建生产者或消息的TopicAndFilter须根据生产者的信息(系统大量使用TopicAndFilter为map键,构建绝对不能出错) + * @author youngforever + * + */ +public class TopicAndFilter implements Serializable{ + private static final long serialVersionUID = -7983983900930009841L; + private String topic; + private Map filter; + + public TopicAndFilter(){}; + + public TopicAndFilter(String topic, Map filter) { + if (topic == null) { + throw new IllegalArgumentException("Left value is not effective."); + } + //允许filter为null +// if (filter == null) { +// throw new IllegalArgumentException("Right value is not effective."); +// } + this.topic = topic; + this.filter = filter; + } +// public TopicAndFilter(String topicAndFilter) { +// +// } + + public String getTopic() { + return topic; + } + + public Map getFilter() { + return filter; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((topic == null) ? 0 : topic.hashCode()); + result = prime * result + ((filter == null) ? 0 : filter.hashCode()); + return result; + } +// public int hashCode() { +// if(right!=null){ +// return (left.toString()+right.toString()).hashCode(); +// }else return left.toString().hashCode(); +// } + + @SuppressWarnings("unchecked") + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (!(obj instanceof TopicAndFilter)) + return false; + TopicAndFilter other = (TopicAndFilter) obj; + if (topic == null) { + if (other.topic != null) + return false; + } else if (!topic.equals(other.topic)) + return false; + if (filter == null) { + if (other.filter != null) + return false; + } else if (!filter.equals(other.filter)) + return false; + return true; + } + + @Override + public String toString() { + if(filter!=null) + return topic + filter.toString() ; + else return topic +"null"; + } + +} diff --git a/src/main/java/com/alibaba/middleware/race/mom/util/Triple.java b/src/main/java/com/alibaba/middleware/race/mom/util/Triple.java new file mode 100644 index 0000000..86ae2b6 --- /dev/null +++ b/src/main/java/com/alibaba/middleware/race/mom/util/Triple.java @@ -0,0 +1,94 @@ +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package com.alibaba.middleware.race.mom.util; + +public class Triple { + + private final L left; + private final M middle; + private final R right; + + public Triple(L left, M middle, R right) { + if (left == null) { + throw new IllegalArgumentException("Left value is not effective."); + } + if (middle == null) { + throw new IllegalArgumentException("Middle value is not effective."); + } + if (right == null) { + throw new IllegalArgumentException("Right value is not effective."); + } + this.left = left; + this.middle = middle; + this.right = right; + } + + public L getLeft() { + return this.left; + } + + public M getMiddle() { + return this.middle; + } + + public R getRight() { + return this.right; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((left == null) ? 0 : left.hashCode()); + result = prime * result + ((middle == null) ? 0 : middle.hashCode()); + result = prime * result + ((right == null) ? 0 : right.hashCode()); + return result; + } + + @SuppressWarnings("unchecked") + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Triple other = (Triple) obj; + if (left == null) { + if (other.left != null) + return false; + } else if (!left.equals(other.left)) + return false; + if (middle == null) { + if (other.middle != null) + return false; + } else if (!middle.equals(other.middle)) + return false; + if (right == null) { + if (other.right != null) + return false; + } else if (!right.equals(other.right)) + return false; + return true; + } + + @Override + public String toString() { + return "<" + left + "," + middle + "," + right + ">"; + } + +} diff --git a/src/main/java/log4j.properties b/src/main/java/log4j.properties new file mode 100644 index 0000000..cead30e --- /dev/null +++ b/src/main/java/log4j.properties @@ -0,0 +1,93 @@ +log4j.rootCategory=error,stdout +log4j.addivity.org.apache=true + +#stout 控制台打印 +log4j.appender.stdout.Encoding=UTF-8 +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +#log4j.appender.stdout.layout.ConversionPattern=[QC] %-d{yyyy-MM-dd HH:mm:ss} %p [%t] %C.%M(%L) | %m%n +log4j.appender.stdout.layout.ConversionPattern= %p %C.%M(%L) | %m%n + +#自定义打印到文件 +log4j.appender.R=org.apache.log4j.DailyRollingFileAppender +log4j.appender.R.File=mom.log +log4j.appender.R.layout=org.apache.log4j.PatternLayout +log4j.appender.R.layout.ConversionPattern=%d-[TS] %p %t %c - %m%n + + ## Disable other log 不打印的包日记 +log4j.logger.io.netty=OFF +log4j.logger.org.springframework=OFF +log4j.logger.org.apache.struts2=OFF +log4j.logger.com.opensymphony.xwork2=OFF +log4j.logger.com.ibatis=OFF +log4j.logger.org.hibernate=OFF + + +#暂时没有使用下面的logger +# 应用于控制台 +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.Threshold=INFO +log4j.appender.CONSOLE.Target=System.out +log4j.appender.CONSOLE.Encoding=GBK +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n + +# 每天新建日志 +log4j.appender.A1=org.apache.log4j.DailyRollingFileAppender +log4j.appender.A1.File=C\:/log4j/log +log4j.appender.A1.Encoding=GBK +log4j.appender.A1.Threshold=DEBUG +log4j.appender.A1.DatePattern='.'yyyy-MM-dd +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}\:%L \: %m%n + +#应用于文件 +log4j.appender.FILE=org.apache.log4j.FileAppender +log4j.appender.FILE.File=C\:/log4j/file.log +log4j.appender.FILE.Append=false +log4j.appender.FILE.Encoding=GBK +log4j.appender.FILE.layout=org.apache.log4j.PatternLayout +log4j.appender.FILE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n + +# 应用于文件回滚 +log4j.appender.ROLLING_FILE=org.apache.log4j.RollingFileAppender +log4j.appender.ROLLING_FILE.Threshold=ERROR +log4j.appender.ROLLING_FILE.File=rolling.log +log4j.appender.ROLLING_FILE.Append=true +log4j.appender.CONSOLE_FILE.Encoding=GBK +log4j.appender.ROLLING_FILE.MaxFileSize=10KB +log4j.appender.ROLLING_FILE.MaxBackupIndex=1 +log4j.appender.ROLLING_FILE.layout=org.apache.log4j.PatternLayout +log4j.appender.ROLLING_FILE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n + +#自定义Appender +log4j.appender.im = net.cybercorlin.util.logger.appender.IMAppender +log4j.appender.im.host = mail.cybercorlin.net +log4j.appender.im.username = username +log4j.appender.im.password = password +log4j.appender.im.recipient = yyflyons@163.com +log4j.appender.im.layout=org.apache.log4j.PatternLayout +log4j.appender.im.layout.ConversionPattern =[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n + +#应用于socket +log4j.appender.SOCKET=org.apache.log4j.RollingFileAppender +log4j.appender.SOCKET.RemoteHost=localhost +log4j.appender.SOCKET.Port=5001 +log4j.appender.SOCKET.LocationInfo=true +# Set up for Log Facter 5 +log4j.appender.SOCKET.layout=org.apache.log4j.PatternLayout +log4j.appender.SOCET.layout.ConversionPattern=[start]%d{DATE}[DATE]%n%p[PRIORITY]%n%x[NDC]%n%t[THREAD]%n%c[CATEGORY]%n%m[MESSAGE]%n%n +# Log Factor 5 Appender +log4j.appender.LF5_APPENDER=org.apache.log4j.lf5.LF5Appender +log4j.appender.LF5_APPENDER.MaxNumberOfRecords=2000 + +# 发送日志给邮件 +log4j.appender.MAIL=org.apache.log4j.net.SMTPAppender +log4j.appender.MAIL.Threshold=FATAL +log4j.appender.MAIL.BufferSize=10 +log4j.appender.MAIL.From=yyflyons@163.com +log4j.appender.MAIL.SMTPHost=www.wusetu.com +log4j.appender.MAIL.Subject=Log4J Message +log4j.appender.MAIL.To=yyflyons@126.com +log4j.appender.MAIL.layout=org.apache.log4j.PatternLayout +log4j.appender.MAIL.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n diff --git a/src/test/java/com/alibaba/middleware/race/mom/AppTest.java b/src/test/java/com/alibaba/middleware/race/mom/AppTest.java new file mode 100644 index 0000000..a77f694 --- /dev/null +++ b/src/test/java/com/alibaba/middleware/race/mom/AppTest.java @@ -0,0 +1,40 @@ +package com.alibaba.middleware.race.mom; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + +/** + * Unit test for simple App. + */ +public class AppTest { + private static ExecutorService executorService=Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + public static void main(String args[]) throws InterruptedException{ + long start=System.currentTimeMillis(); + for (int i = 0; i properties = new HashMap(); + public String getProperty(String key) { + return properties.get(key); + } + /** + * 设置消息属性 + * @param key + * @param value + */ + public void setProperty(String key, String value) { + properties.put(key, value); + } + /** + * 删除消息属性 + * @param key + */ + public void removeProperty(String key) { + properties.remove(key); + } + public Map getProperties() { + return properties; + } + + List user = new ArrayList(); + + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public List getUser() { + return user; + } + + public void setUser(List user) { + this.user = user; + } +} + +class User { + long id; + String name; + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} + +public class fstjson { + public static void main(String[] args) { + Group group = new Group(); + group.setId(0L); + group.setName("admin"); + group.setProperty("1", "1"); + User guestUser = new User(); + guestUser.setId(2L); + guestUser.setName("guest"); + + User rootUser = new User(); + rootUser.setId(3L); + rootUser.setName("root"); + + group.getUser().add(guestUser); + group.getUser().add(rootUser); + String jsonString = JSON.toJSONString(group); + System.out.println(jsonString); + } +} diff --git a/src/test/java/com/alibaba/middleware/race/mom/test.java b/src/test/java/com/alibaba/middleware/race/mom/test.java new file mode 100644 index 0000000..8ed6915 --- /dev/null +++ b/src/test/java/com/alibaba/middleware/race/mom/test.java @@ -0,0 +1,47 @@ +package com.alibaba.middleware.race.mom; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; + +public class test extends Thread{ + public static BlockingQueue queue=new LinkedBlockingQueue(3); + private int index; + public test(int i){ + this.index=i; + } + + public void run(){ + try{ + queue.put(String.valueOf(this.index)); + System.out.println("put {"+this.index+"} into queue!"); + }catch(Exception e){ + e.printStackTrace(); + } + } + + public static void main(String args[]){ + ExecutorService service=Executors.newCachedThreadPool(); + for( int i=0; i<10; i++){ + service.submit(new test(i)); + } + Thread thread = new Thread(){ + public void run(){ + try{ + while(true){ + Thread.sleep((int)(Math.random()*1000)); + // if(test.queue.isEmpty()) break; + String str=test.queue.take(); + System.out.println("take {" + str+"} out of queue!"); + } + }catch(Exception e){ + e.printStackTrace(); + } + } + }; + service.submit(thread); + service.shutdown(); + } + +} diff --git a/target/classes/assembly.xml b/target/classes/assembly.xml new file mode 100644 index 0000000..173e76c --- /dev/null +++ b/target/classes/assembly.xml @@ -0,0 +1,30 @@ + + bin + middleware-mom + true + + dir + + + + ${project.basedir} + / + + README* + LICENSE* + NOTICE* + + + + ${project.basedir}/lib + lib + + + + + /lib + true + runtime + + + \ No newline at end of file diff --git a/target/classes/com/alibaba/middleware/race/mom/Broker$1$1.class b/target/classes/com/alibaba/middleware/race/mom/Broker$1$1.class new file mode 100644 index 0000000..e137dfa Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/Broker$1$1.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/Broker$1.class b/target/classes/com/alibaba/middleware/race/mom/Broker$1.class new file mode 100644 index 0000000..0649020 Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/Broker$1.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/Broker$2.class b/target/classes/com/alibaba/middleware/race/mom/Broker$2.class new file mode 100644 index 0000000..5d95470 Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/Broker$2.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/Broker$3.class b/target/classes/com/alibaba/middleware/race/mom/Broker$3.class new file mode 100644 index 0000000..2655db7 Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/Broker$3.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/Broker$4.class b/target/classes/com/alibaba/middleware/race/mom/Broker$4.class new file mode 100644 index 0000000..6fe36fa Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/Broker$4.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/Broker.class b/target/classes/com/alibaba/middleware/race/mom/Broker.class new file mode 100644 index 0000000..7aa4569 Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/Broker.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/ConsumeResult.class b/target/classes/com/alibaba/middleware/race/mom/ConsumeResult.class new file mode 100644 index 0000000..e519425 Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/ConsumeResult.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/ConsumeStatus.class b/target/classes/com/alibaba/middleware/race/mom/ConsumeStatus.class new file mode 100644 index 0000000..b736376 Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/ConsumeStatus.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/Consumer.class b/target/classes/com/alibaba/middleware/race/mom/Consumer.class new file mode 100644 index 0000000..cac9d3f Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/Consumer.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/DefaultConsumer$1$1.class b/target/classes/com/alibaba/middleware/race/mom/DefaultConsumer$1$1.class new file mode 100644 index 0000000..fd080e9 Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/DefaultConsumer$1$1.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/DefaultConsumer$1.class b/target/classes/com/alibaba/middleware/race/mom/DefaultConsumer$1.class new file mode 100644 index 0000000..effcde9 Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/DefaultConsumer$1.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/DefaultConsumer.class b/target/classes/com/alibaba/middleware/race/mom/DefaultConsumer.class new file mode 100644 index 0000000..bf31c57 Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/DefaultConsumer.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/DefaultProducer$1$1.class b/target/classes/com/alibaba/middleware/race/mom/DefaultProducer$1$1.class new file mode 100644 index 0000000..90dfe6d Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/DefaultProducer$1$1.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/DefaultProducer$1.class b/target/classes/com/alibaba/middleware/race/mom/DefaultProducer$1.class new file mode 100644 index 0000000..61540f9 Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/DefaultProducer$1.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/DefaultProducer.class b/target/classes/com/alibaba/middleware/race/mom/DefaultProducer.class new file mode 100644 index 0000000..cd182bd Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/DefaultProducer.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/Message.class b/target/classes/com/alibaba/middleware/race/mom/Message.class new file mode 100644 index 0000000..98c4c30 Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/Message.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/MessageListener.class b/target/classes/com/alibaba/middleware/race/mom/MessageListener.class new file mode 100644 index 0000000..9fb44c1 Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/MessageListener.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/Producer.class b/target/classes/com/alibaba/middleware/race/mom/Producer.class new file mode 100644 index 0000000..fd18e51 Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/Producer.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/SendCallback.class b/target/classes/com/alibaba/middleware/race/mom/SendCallback.class new file mode 100644 index 0000000..de83499 Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/SendCallback.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/SendResult.class b/target/classes/com/alibaba/middleware/race/mom/SendResult.class new file mode 100644 index 0000000..2a13486 Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/SendResult.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/SendStatus.class b/target/classes/com/alibaba/middleware/race/mom/SendStatus.class new file mode 100644 index 0000000..e968bb2 Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/SendStatus.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/serializer/JdkObjectSerializer.class b/target/classes/com/alibaba/middleware/race/mom/serializer/JdkObjectSerializer.class new file mode 100644 index 0000000..e0cb12e Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/serializer/JdkObjectSerializer.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/serializer/KryoxSerializer.class b/target/classes/com/alibaba/middleware/race/mom/serializer/KryoxSerializer.class new file mode 100644 index 0000000..1e0cc17 Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/serializer/KryoxSerializer.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/serializer/RpcDecoder.class b/target/classes/com/alibaba/middleware/race/mom/serializer/RpcDecoder.class new file mode 100644 index 0000000..b69893a Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/serializer/RpcDecoder.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/serializer/RpcEncoder.class b/target/classes/com/alibaba/middleware/race/mom/serializer/RpcEncoder.class new file mode 100644 index 0000000..05f844a Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/serializer/RpcEncoder.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/util/Info.class b/target/classes/com/alibaba/middleware/race/mom/util/Info.class new file mode 100644 index 0000000..638a324 Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/util/Info.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/util/InfoBodyConsumer.class b/target/classes/com/alibaba/middleware/race/mom/util/InfoBodyConsumer.class new file mode 100644 index 0000000..c37e109 Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/util/InfoBodyConsumer.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/util/InfoBodyProduce.class b/target/classes/com/alibaba/middleware/race/mom/util/InfoBodyProduce.class new file mode 100644 index 0000000..70ecee9 Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/util/InfoBodyProduce.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/util/InfoType.class b/target/classes/com/alibaba/middleware/race/mom/util/InfoType.class new file mode 100644 index 0000000..c7e6dfa Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/util/InfoType.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/util/Pair.class b/target/classes/com/alibaba/middleware/race/mom/util/Pair.class new file mode 100644 index 0000000..d12a26f Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/util/Pair.class differ diff --git a/target/classes/com/alibaba/middleware/race/mom/util/Triple.class b/target/classes/com/alibaba/middleware/race/mom/util/Triple.class new file mode 100644 index 0000000..d7050a6 Binary files /dev/null and b/target/classes/com/alibaba/middleware/race/mom/util/Triple.class differ diff --git a/target/classes/log4j.properties b/target/classes/log4j.properties new file mode 100644 index 0000000..e95ba95 --- /dev/null +++ b/target/classes/log4j.properties @@ -0,0 +1,93 @@ +log4j.rootCategory=error, stdout +log4j.addivity.org.apache=true + +#stout 控制台打印 +log4j.appender.stdout.Encoding=UTF-8 +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +#log4j.appender.stdout.layout.ConversionPattern=[QC] %-d{yyyy-MM-dd HH:mm:ss} %p [%t] %C.%M(%L) | %m%n +log4j.appender.stdout.layout.ConversionPattern= %p %C.%M(%L) | %m%n + +#自定义打印到文件 +log4j.appender.R=org.apache.log4j.DailyRollingFileAppender +log4j.appender.R.File=mom.log +log4j.appender.R.layout=org.apache.log4j.PatternLayout +log4j.appender.R.layout.ConversionPattern=%d-[TS] %p %t %c - %m%n + + ## Disable other log 不打印的包日记 +log4j.logger.io.netty=OFF +log4j.logger.org.springframework=OFF +log4j.logger.org.apache.struts2=OFF +log4j.logger.com.opensymphony.xwork2=OFF +log4j.logger.com.ibatis=OFF +log4j.logger.org.hibernate=OFF + + +#暂时没有使用下面的logger +# 应用于控制台 +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.Threshold=INFO +log4j.appender.CONSOLE.Target=System.out +log4j.appender.CONSOLE.Encoding=GBK +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n + +# 每天新建日志 +log4j.appender.A1=org.apache.log4j.DailyRollingFileAppender +log4j.appender.A1.File=C\:/log4j/log +log4j.appender.A1.Encoding=GBK +log4j.appender.A1.Threshold=DEBUG +log4j.appender.A1.DatePattern='.'yyyy-MM-dd +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}\:%L \: %m%n + +#应用于文件 +log4j.appender.FILE=org.apache.log4j.FileAppender +log4j.appender.FILE.File=C\:/log4j/file.log +log4j.appender.FILE.Append=false +log4j.appender.FILE.Encoding=GBK +log4j.appender.FILE.layout=org.apache.log4j.PatternLayout +log4j.appender.FILE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n + +# 应用于文件回滚 +log4j.appender.ROLLING_FILE=org.apache.log4j.RollingFileAppender +log4j.appender.ROLLING_FILE.Threshold=ERROR +log4j.appender.ROLLING_FILE.File=rolling.log +log4j.appender.ROLLING_FILE.Append=true +log4j.appender.CONSOLE_FILE.Encoding=GBK +log4j.appender.ROLLING_FILE.MaxFileSize=10KB +log4j.appender.ROLLING_FILE.MaxBackupIndex=1 +log4j.appender.ROLLING_FILE.layout=org.apache.log4j.PatternLayout +log4j.appender.ROLLING_FILE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n + +#自定义Appender +log4j.appender.im = net.cybercorlin.util.logger.appender.IMAppender +log4j.appender.im.host = mail.cybercorlin.net +log4j.appender.im.username = username +log4j.appender.im.password = password +log4j.appender.im.recipient = yyflyons@163.com +log4j.appender.im.layout=org.apache.log4j.PatternLayout +log4j.appender.im.layout.ConversionPattern =[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n + +#应用于socket +log4j.appender.SOCKET=org.apache.log4j.RollingFileAppender +log4j.appender.SOCKET.RemoteHost=localhost +log4j.appender.SOCKET.Port=5001 +log4j.appender.SOCKET.LocationInfo=true +# Set up for Log Facter 5 +log4j.appender.SOCKET.layout=org.apache.log4j.PatternLayout +log4j.appender.SOCET.layout.ConversionPattern=[start]%d{DATE}[DATE]%n%p[PRIORITY]%n%x[NDC]%n%t[THREAD]%n%c[CATEGORY]%n%m[MESSAGE]%n%n +# Log Factor 5 Appender +log4j.appender.LF5_APPENDER=org.apache.log4j.lf5.LF5Appender +log4j.appender.LF5_APPENDER.MaxNumberOfRecords=2000 + +# 发送日志给邮件 +log4j.appender.MAIL=org.apache.log4j.net.SMTPAppender +log4j.appender.MAIL.Threshold=FATAL +log4j.appender.MAIL.BufferSize=10 +log4j.appender.MAIL.From=yyflyons@163.com +log4j.appender.MAIL.SMTPHost=www.wusetu.com +log4j.appender.MAIL.Subject=Log4J Message +log4j.appender.MAIL.To=yyflyons@126.com +log4j.appender.MAIL.layout=org.apache.log4j.PatternLayout +log4j.appender.MAIL.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n \ No newline at end of file diff --git a/target/lib/archive-tmp/middleware-mom-1.0.jar b/target/lib/archive-tmp/middleware-mom-1.0.jar new file mode 100644 index 0000000..315a206 Binary files /dev/null and b/target/lib/archive-tmp/middleware-mom-1.0.jar differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/Broker$1$1.class b/target/lib/classes/com/alibaba/middleware/race/mom/Broker$1$1.class new file mode 100644 index 0000000..200f236 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/Broker$1$1.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/Broker$1.class b/target/lib/classes/com/alibaba/middleware/race/mom/Broker$1.class new file mode 100644 index 0000000..8805eeb Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/Broker$1.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/Broker.class b/target/lib/classes/com/alibaba/middleware/race/mom/Broker.class new file mode 100644 index 0000000..b0d2181 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/Broker.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/ConsumeResult.class b/target/lib/classes/com/alibaba/middleware/race/mom/ConsumeResult.class new file mode 100644 index 0000000..08ea249 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/ConsumeResult.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/ConsumeStatus.class b/target/lib/classes/com/alibaba/middleware/race/mom/ConsumeStatus.class new file mode 100644 index 0000000..d35395a Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/ConsumeStatus.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/Consumer.class b/target/lib/classes/com/alibaba/middleware/race/mom/Consumer.class new file mode 100644 index 0000000..68dbdb9 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/Consumer.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/DefaultConsumer$1$1.class b/target/lib/classes/com/alibaba/middleware/race/mom/DefaultConsumer$1$1.class new file mode 100644 index 0000000..3e93986 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/DefaultConsumer$1$1.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/DefaultConsumer$1.class b/target/lib/classes/com/alibaba/middleware/race/mom/DefaultConsumer$1.class new file mode 100644 index 0000000..500eda8 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/DefaultConsumer$1.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/DefaultConsumer.class b/target/lib/classes/com/alibaba/middleware/race/mom/DefaultConsumer.class new file mode 100644 index 0000000..a261512 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/DefaultConsumer.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/DefaultProducer$1$1.class b/target/lib/classes/com/alibaba/middleware/race/mom/DefaultProducer$1$1.class new file mode 100644 index 0000000..fcdb89d Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/DefaultProducer$1$1.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/DefaultProducer$1.class b/target/lib/classes/com/alibaba/middleware/race/mom/DefaultProducer$1.class new file mode 100644 index 0000000..833c59a Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/DefaultProducer$1.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/DefaultProducer$2.class b/target/lib/classes/com/alibaba/middleware/race/mom/DefaultProducer$2.class new file mode 100644 index 0000000..7a7d7a1 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/DefaultProducer$2.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/DefaultProducer.class b/target/lib/classes/com/alibaba/middleware/race/mom/DefaultProducer.class new file mode 100644 index 0000000..7e0be60 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/DefaultProducer.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/Message.class b/target/lib/classes/com/alibaba/middleware/race/mom/Message.class new file mode 100644 index 0000000..add5f4a Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/Message.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/MessageListener.class b/target/lib/classes/com/alibaba/middleware/race/mom/MessageListener.class new file mode 100644 index 0000000..0469138 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/MessageListener.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/Producer.class b/target/lib/classes/com/alibaba/middleware/race/mom/Producer.class new file mode 100644 index 0000000..831c5c4 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/Producer.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/QueueMessage.class b/target/lib/classes/com/alibaba/middleware/race/mom/QueueMessage.class new file mode 100644 index 0000000..22019dd Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/QueueMessage.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/SendCallback.class b/target/lib/classes/com/alibaba/middleware/race/mom/SendCallback.class new file mode 100644 index 0000000..674ae37 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/SendCallback.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/SendResult.class b/target/lib/classes/com/alibaba/middleware/race/mom/SendResult.class new file mode 100644 index 0000000..b719a4e Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/SendResult.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/SendStatus.class b/target/lib/classes/com/alibaba/middleware/race/mom/SendStatus.class new file mode 100644 index 0000000..0235752 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/SendStatus.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/AccumulateHandler$1.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/AccumulateHandler$1.class new file mode 100644 index 0000000..6c2de6f Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/AccumulateHandler$1.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/AccumulateHandler.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/AccumulateHandler.class new file mode 100644 index 0000000..a94dd9b Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/AccumulateHandler.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/MessageInfo.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/MessageInfo.class new file mode 100644 index 0000000..9473061 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/MessageInfo.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/MessageInfoQueue.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/MessageInfoQueue.class new file mode 100644 index 0000000..47fae0c Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/MessageInfoQueue.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/MessageManager.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/MessageManager.class new file mode 100644 index 0000000..9e6d5f7 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/MessageManager.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/MessageSend.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/MessageSend.class new file mode 100644 index 0000000..ea810e2 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/MessageSend.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/MessageStore$1.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/MessageStore$1.class new file mode 100644 index 0000000..c14988c Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/MessageStore$1.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/MessageStore$2.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/MessageStore$2.class new file mode 100644 index 0000000..0610a0e Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/MessageStore$2.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/MessageStore.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/MessageStore.class new file mode 100644 index 0000000..c649aa5 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/MessageStore.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/Offset.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/Offset.class new file mode 100644 index 0000000..b9fa317 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/Offset.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/Recover.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/Recover.class new file mode 100644 index 0000000..4e41850 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/Recover.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/SendInfo.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/SendInfo.class new file mode 100644 index 0000000..742a9c4 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/SendInfo.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/SendInfoScanner$1$1.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/SendInfoScanner$1$1.class new file mode 100644 index 0000000..34db144 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/SendInfoScanner$1$1.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/SendInfoScanner$1.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/SendInfoScanner$1.class new file mode 100644 index 0000000..e6f81db Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/SendInfoScanner$1.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/SendInfoScanner$2.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/SendInfoScanner$2.class new file mode 100644 index 0000000..f29e66b Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/SendInfoScanner$2.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/SendInfoScanner.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/SendInfoScanner.class new file mode 100644 index 0000000..591d95e Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/SendInfoScanner.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/StoreSucceedListener$1.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/StoreSucceedListener$1.class new file mode 100644 index 0000000..46f2166 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/StoreSucceedListener$1.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/StoreSucceedListener.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/StoreSucceedListener.class new file mode 100644 index 0000000..82bf73e Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/StoreSucceedListener.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/SubsStore$1.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/SubsStore$1.class new file mode 100644 index 0000000..d45396d Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/SubsStore$1.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/SubsStore.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/SubsStore.class new file mode 100644 index 0000000..0eca921 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/function/SubsStore.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/group/ConsumerGroup$1.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/group/ConsumerGroup$1.class new file mode 100644 index 0000000..0e688c3 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/group/ConsumerGroup$1.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/group/ConsumerGroup$2.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/group/ConsumerGroup$2.class new file mode 100644 index 0000000..291889c Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/group/ConsumerGroup$2.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/group/ConsumerGroup$3.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/group/ConsumerGroup$3.class new file mode 100644 index 0000000..63c39a4 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/group/ConsumerGroup$3.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/group/ConsumerGroup.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/group/ConsumerGroup.class new file mode 100644 index 0000000..216b847 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/group/ConsumerGroup.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/broker/group/ProducerGroup.class b/target/lib/classes/com/alibaba/middleware/race/mom/broker/group/ProducerGroup.class new file mode 100644 index 0000000..8571e31 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/broker/group/ProducerGroup.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/config/Config.class b/target/lib/classes/com/alibaba/middleware/race/mom/config/Config.class new file mode 100644 index 0000000..4b416ca Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/config/Config.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/serializer/JdkObjectSerializer.class b/target/lib/classes/com/alibaba/middleware/race/mom/serializer/JdkObjectSerializer.class new file mode 100644 index 0000000..1804687 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/serializer/JdkObjectSerializer.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/serializer/KryoxSerializer.class b/target/lib/classes/com/alibaba/middleware/race/mom/serializer/KryoxSerializer.class new file mode 100644 index 0000000..d671081 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/serializer/KryoxSerializer.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/serializer/RpcDecoder.class b/target/lib/classes/com/alibaba/middleware/race/mom/serializer/RpcDecoder.class new file mode 100644 index 0000000..474e732 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/serializer/RpcDecoder.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/serializer/RpcEncoder.class b/target/lib/classes/com/alibaba/middleware/race/mom/serializer/RpcEncoder.class new file mode 100644 index 0000000..4c24744 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/serializer/RpcEncoder.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/ArrayStore.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/ArrayStore.class new file mode 100644 index 0000000..fef6c3f Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/ArrayStore.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/ArrayStoreImp.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/ArrayStoreImp.class new file mode 100644 index 0000000..088a0c5 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/ArrayStoreImp.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/Comm.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/Comm.class new file mode 100644 index 0000000..81911c3 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/Comm.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/DataMappedFile.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/DataMappedFile.class new file mode 100644 index 0000000..112f55f Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/DataMappedFile.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/FileStoreManager$1.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/FileStoreManager$1.class new file mode 100644 index 0000000..894b285 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/FileStoreManager$1.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/FileStoreManager$2.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/FileStoreManager$2.class new file mode 100644 index 0000000..6b5cc49 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/FileStoreManager$2.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/FileStoreManager.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/FileStoreManager.class new file mode 100644 index 0000000..56f2ef8 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/FileStoreManager.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/IndexMappedFile.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/IndexMappedFile.class new file mode 100644 index 0000000..2f976bc Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/IndexMappedFile.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/MappedFileInfo.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/MappedFileInfo.class new file mode 100644 index 0000000..8ae9e40 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/MappedFileInfo.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/Message.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/Message.class new file mode 100644 index 0000000..1ab6176 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/Message.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/MsgFileUtil.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/MsgFileUtil.class new file mode 100644 index 0000000..faa22b9 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/MsgFileUtil.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/MsgStore.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/MsgStore.class new file mode 100644 index 0000000..4e7849d Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/MsgStore.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/MsgStoreImp_MappedBuffer2.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/MsgStoreImp_MappedBuffer2.class new file mode 100644 index 0000000..34cc897 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/MsgStoreImp_MappedBuffer2.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/OffsetNum.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/OffsetNum.class new file mode 100644 index 0000000..4157df6 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/OffsetNum.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/Redo$RedoBean.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/Redo$RedoBean.class new file mode 100644 index 0000000..3639e59 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/Redo$RedoBean.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/Redo.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/Redo.class new file mode 100644 index 0000000..53647f9 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/Redo.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/RedoLog$LogBean.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/RedoLog$LogBean.class new file mode 100644 index 0000000..5a9b788 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/RedoLog$LogBean.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/RedoLog.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/RedoLog.class new file mode 100644 index 0000000..b205fa0 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/RedoLog.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/StoreConfig.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/StoreConfig.class new file mode 100644 index 0000000..3ec3b06 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/StoreConfig.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/StoreMsg.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/StoreMsg.class new file mode 100644 index 0000000..6635079 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/StoreMsg.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/SubscribeStore.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/SubscribeStore.class new file mode 100644 index 0000000..9dae89b Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/SubscribeStore.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/SubscribeStoreImp.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/SubscribeStoreImp.class new file mode 100644 index 0000000..8b23ce7 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/SubscribeStoreImp.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/SubscribeStoreImp_MappedBuffer2.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/SubscribeStoreImp_MappedBuffer2.class new file mode 100644 index 0000000..03f95d0 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/SubscribeStoreImp_MappedBuffer2.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/store/redotest.class b/target/lib/classes/com/alibaba/middleware/race/mom/store/redotest.class new file mode 100644 index 0000000..9848cbc Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/store/redotest.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/util/ConsumerInfo.class b/target/lib/classes/com/alibaba/middleware/race/mom/util/ConsumerInfo.class new file mode 100644 index 0000000..7e41553 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/util/ConsumerInfo.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/util/Info.class b/target/lib/classes/com/alibaba/middleware/race/mom/util/Info.class new file mode 100644 index 0000000..d91b7f8 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/util/Info.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/util/InfoBodyConsumer.class b/target/lib/classes/com/alibaba/middleware/race/mom/util/InfoBodyConsumer.class new file mode 100644 index 0000000..eb94b3a Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/util/InfoBodyConsumer.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/util/InfoBodyProduceList.class b/target/lib/classes/com/alibaba/middleware/race/mom/util/InfoBodyProduceList.class new file mode 100644 index 0000000..4c7fa7b Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/util/InfoBodyProduceList.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/util/InfoType.class b/target/lib/classes/com/alibaba/middleware/race/mom/util/InfoType.class new file mode 100644 index 0000000..71d4de4 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/util/InfoType.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/util/InfoUtil.class b/target/lib/classes/com/alibaba/middleware/race/mom/util/InfoUtil.class new file mode 100644 index 0000000..a4d00e8 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/util/InfoUtil.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/util/TopicAndFilter.class b/target/lib/classes/com/alibaba/middleware/race/mom/util/TopicAndFilter.class new file mode 100644 index 0000000..d3f38e5 Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/util/TopicAndFilter.class differ diff --git a/target/lib/classes/com/alibaba/middleware/race/mom/util/Triple.class b/target/lib/classes/com/alibaba/middleware/race/mom/util/Triple.class new file mode 100644 index 0000000..c90e21c Binary files /dev/null and b/target/lib/classes/com/alibaba/middleware/race/mom/util/Triple.class differ diff --git a/target/lib/maven-archiver/pom.properties b/target/lib/maven-archiver/pom.properties new file mode 100644 index 0000000..5fb9d4c --- /dev/null +++ b/target/lib/maven-archiver/pom.properties @@ -0,0 +1,5 @@ +#Generated by Maven +#Thu Aug 20 17:30:42 CST 2015 +version=1.0 +groupId=com.alibaba.race +artifactId=middleware-mom diff --git a/target/lib/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst b/target/lib/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst new file mode 100644 index 0000000..54f85eb --- /dev/null +++ b/target/lib/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst @@ -0,0 +1,82 @@ +com/alibaba/middleware/race/mom/store/MsgStoreImp_MappedBuffer2.class +com/alibaba/middleware/race/mom/config/Config.class +com/alibaba/middleware/race/mom/ConsumeResult.class +com/alibaba/middleware/race/mom/serializer/JdkObjectSerializer.class +com/alibaba/middleware/race/mom/DefaultProducer$1.class +com/alibaba/middleware/race/mom/util/InfoType.class +com/alibaba/middleware/race/mom/SendStatus.class +com/alibaba/middleware/race/mom/serializer/RpcDecoder.class +com/alibaba/middleware/race/mom/QueueMessage.class +com/alibaba/middleware/race/mom/broker/function/MessageInfo.class +com/alibaba/middleware/race/mom/broker/group/ConsumerGroup.class +com/alibaba/middleware/race/mom/store/RedoLog$LogBean.class +com/alibaba/middleware/race/mom/Message.class +com/alibaba/middleware/race/mom/broker/function/SendInfoScanner$1.class +com/alibaba/middleware/race/mom/store/OffsetNum.class +com/alibaba/middleware/race/mom/util/InfoBodyConsumer.class +com/alibaba/middleware/race/mom/broker/function/Offset.class +com/alibaba/middleware/race/mom/DefaultConsumer.class +com/alibaba/middleware/race/mom/broker/group/ConsumerGroup$1.class +com/alibaba/middleware/race/mom/broker/function/AccumulateHandler$1.class +com/alibaba/middleware/race/mom/broker/function/AccumulateHandler.class +com/alibaba/middleware/race/mom/util/Info.class +com/alibaba/middleware/race/mom/broker/group/ConsumerGroup$3.class +com/alibaba/middleware/race/mom/store/ArrayStore.class +com/alibaba/middleware/race/mom/store/FileStoreManager.class +com/alibaba/middleware/race/mom/store/DataMappedFile.class +com/alibaba/middleware/race/mom/store/SubscribeStoreImp_MappedBuffer2.class +com/alibaba/middleware/race/mom/broker/function/MessageStore$1.class +com/alibaba/middleware/race/mom/store/FileStoreManager$2.class +com/alibaba/middleware/race/mom/Broker$1.class +com/alibaba/middleware/race/mom/broker/function/SubsStore.class +com/alibaba/middleware/race/mom/store/IndexMappedFile.class +com/alibaba/middleware/race/mom/serializer/RpcEncoder.class +com/alibaba/middleware/race/mom/store/FileStoreManager$1.class +com/alibaba/middleware/race/mom/ConsumeStatus.class +com/alibaba/middleware/race/mom/broker/function/Recover.class +com/alibaba/middleware/race/mom/broker/function/MessageInfoQueue.class +com/alibaba/middleware/race/mom/serializer/KryoxSerializer.class +com/alibaba/middleware/race/mom/DefaultProducer$1$1.class +com/alibaba/middleware/race/mom/broker/function/MessageSend.class +com/alibaba/middleware/race/mom/broker/group/ConsumerGroup$2.class +com/alibaba/middleware/race/mom/broker/function/SubsStore$1.class +com/alibaba/middleware/race/mom/util/InfoUtil.class +com/alibaba/middleware/race/mom/store/StoreMsg.class +com/alibaba/middleware/race/mom/DefaultConsumer$1.class +com/alibaba/middleware/race/mom/SendResult.class +com/alibaba/middleware/race/mom/store/Redo.class +com/alibaba/middleware/race/mom/broker/function/SendInfoScanner.class +com/alibaba/middleware/race/mom/broker/function/MessageStore$2.class +com/alibaba/middleware/race/mom/DefaultProducer.class +com/alibaba/middleware/race/mom/broker/function/SendInfo.class +com/alibaba/middleware/race/mom/broker/function/MessageManager.class +com/alibaba/middleware/race/mom/broker/group/ProducerGroup.class +com/alibaba/middleware/race/mom/MessageListener.class +com/alibaba/middleware/race/mom/broker/function/SendInfoScanner$1$1.class +com/alibaba/middleware/race/mom/util/ConsumerInfo.class +com/alibaba/middleware/race/mom/store/MsgFileUtil.class +com/alibaba/middleware/race/mom/store/ArrayStoreImp.class +com/alibaba/middleware/race/mom/store/StoreConfig.class +com/alibaba/middleware/race/mom/store/SubscribeStoreImp.class +com/alibaba/middleware/race/mom/store/RedoLog.class +com/alibaba/middleware/race/mom/SendCallback.class +com/alibaba/middleware/race/mom/broker/function/SendInfoScanner$2.class +com/alibaba/middleware/race/mom/store/MappedFileInfo.class +com/alibaba/middleware/race/mom/Broker$1$1.class +com/alibaba/middleware/race/mom/DefaultConsumer$1$1.class +com/alibaba/middleware/race/mom/util/Triple.class +com/alibaba/middleware/race/mom/DefaultProducer$2.class +com/alibaba/middleware/race/mom/Producer.class +com/alibaba/middleware/race/mom/store/redotest.class +com/alibaba/middleware/race/mom/store/MsgStore.class +com/alibaba/middleware/race/mom/store/Comm.class +com/alibaba/middleware/race/mom/store/Redo$RedoBean.class +com/alibaba/middleware/race/mom/Broker.class +com/alibaba/middleware/race/mom/broker/function/MessageStore.class +com/alibaba/middleware/race/mom/store/Message.class +com/alibaba/middleware/race/mom/util/TopicAndFilter.class +com/alibaba/middleware/race/mom/store/SubscribeStore.class +com/alibaba/middleware/race/mom/broker/function/StoreSucceedListener$1.class +com/alibaba/middleware/race/mom/util/InfoBodyProduceList.class +com/alibaba/middleware/race/mom/broker/function/StoreSucceedListener.class +com/alibaba/middleware/race/mom/Consumer.class diff --git a/target/lib/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst b/target/lib/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst new file mode 100644 index 0000000..f0297f3 --- /dev/null +++ b/target/lib/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst @@ -0,0 +1,65 @@ +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/Message.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/util/TopicAndFilter.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/store/StoreConfig.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/store/StoreMsg.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/util/InfoType.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/broker/function/Offset.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/ConsumeResult.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/store/Comm.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/broker/function/Recover.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/store/Redo.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/broker/function/SendInfoScanner.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/broker/function/MessageStore.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/SendResult.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/DefaultConsumer.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/store/SubscribeStore.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/broker/function/AccumulateHandler.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/QueueMessage.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/store/RedoLog.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/store/SubscribeStoreImp.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/serializer/KryoxSerializer.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/broker/group/ProducerGroup.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/store/MsgFileUtil.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/store/DataMappedFile.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/Consumer.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/SendStatus.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/store/ArrayStoreImp.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/SendCallback.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/store/SubscribeStoreImp_MappedBuffer2.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/store/redotest.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/DefaultProducer1.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/broker/function/SubsStore.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/store/ArrayStore.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/store/Test.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/store/IndexMappedFile.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/util/Info.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/config/Config.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/store/FileStoreManager.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/DefaultProducer.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/Broker.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/Producer.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/MessageListener.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/ConsumeStatus.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/store/MsgStoreImp_MappedBuffer.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/broker/function/ReSendHandler.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/store/MappedFileInfo.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/util/InfoBodyProduceList.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/broker/function/SendInfo.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/util/ConsumerInfo.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/broker/function/MessageManager.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/serializer/RpcDecoder.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/store/Message.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/store/OffsetNum.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/DefaultProducer2.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/store/MsgStore.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/broker/function/MessageSend.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/broker/group/ConsumerGroup.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/serializer/RpcEncoder.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/broker/function/StoreSucceedListener.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/broker/function/MessageInfo.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/store/MsgStoreImp_MappedBuffer2.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/broker/function/MessageInfoQueue.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/util/InfoBodyConsumer.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/util/Triple.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/util/InfoUtil.java +/home/y/trunk/mom/src/main/java/com/alibaba/middleware/race/mom/serializer/JdkObjectSerializer.java diff --git a/target/lib/maven-status/maven-compiler-plugin/testCompile/default-testCompile/createdFiles.lst b/target/lib/maven-status/maven-compiler-plugin/testCompile/default-testCompile/createdFiles.lst new file mode 100644 index 0000000..5f34595 --- /dev/null +++ b/target/lib/maven-status/maven-compiler-plugin/testCompile/default-testCompile/createdFiles.lst @@ -0,0 +1,7 @@ +com/alibaba/middleware/race/mom/test$1.class +com/alibaba/middleware/race/mom/Group.class +com/alibaba/middleware/race/mom/AppTest$1.class +com/alibaba/middleware/race/mom/AppTest.class +com/alibaba/middleware/race/mom/fstjson.class +com/alibaba/middleware/race/mom/User.class +com/alibaba/middleware/race/mom/test.class diff --git a/target/lib/maven-status/maven-compiler-plugin/testCompile/default-testCompile/inputFiles.lst b/target/lib/maven-status/maven-compiler-plugin/testCompile/default-testCompile/inputFiles.lst new file mode 100644 index 0000000..f712999 --- /dev/null +++ b/target/lib/maven-status/maven-compiler-plugin/testCompile/default-testCompile/inputFiles.lst @@ -0,0 +1,3 @@ +/home/y/trunk/mom/src/test/java/com/alibaba/middleware/race/mom/AppTest.java +/home/y/trunk/mom/src/test/java/com/alibaba/middleware/race/mom/fstjson.java +/home/y/trunk/mom/src/test/java/com/alibaba/middleware/race/mom/test.java diff --git a/target/lib/middleware-mom-1.0.jar b/target/lib/middleware-mom-1.0.jar new file mode 100644 index 0000000..23fd544 Binary files /dev/null and b/target/lib/middleware-mom-1.0.jar differ diff --git a/target/lib/middleware-mom-1.0/middleware-mom/lib/asm-1.0.jar b/target/lib/middleware-mom-1.0/middleware-mom/lib/asm-1.0.jar new file mode 100644 index 0000000..4f23d3f Binary files /dev/null and b/target/lib/middleware-mom-1.0/middleware-mom/lib/asm-1.0.jar differ diff --git a/target/lib/middleware-mom-1.0/middleware-mom/lib/asm-3.3.1.jar b/target/lib/middleware-mom-1.0/middleware-mom/lib/asm-3.3.1.jar new file mode 100644 index 0000000..f50f03f Binary files /dev/null and b/target/lib/middleware-mom-1.0/middleware-mom/lib/asm-3.3.1.jar differ diff --git a/target/lib/middleware-mom-1.0/middleware-mom/lib/asm-5.0.3.jar b/target/lib/middleware-mom-1.0/middleware-mom/lib/asm-5.0.3.jar new file mode 100644 index 0000000..573535b Binary files /dev/null and b/target/lib/middleware-mom-1.0/middleware-mom/lib/asm-5.0.3.jar differ diff --git a/target/lib/middleware-mom-1.0/middleware-mom/lib/fastjson-1.2.5.jar b/target/lib/middleware-mom-1.0/middleware-mom/lib/fastjson-1.2.5.jar new file mode 100644 index 0000000..d4a3f7d Binary files /dev/null and b/target/lib/middleware-mom-1.0/middleware-mom/lib/fastjson-1.2.5.jar differ diff --git a/target/lib/middleware-mom-1.0/middleware-mom/lib/fst-2.34.jar b/target/lib/middleware-mom-1.0/middleware-mom/lib/fst-2.34.jar new file mode 100644 index 0000000..2cb5ca8 Binary files /dev/null and b/target/lib/middleware-mom-1.0/middleware-mom/lib/fst-2.34.jar differ diff --git a/target/lib/middleware-mom-1.0/middleware-mom/lib/jackson-core-2.5.3.jar b/target/lib/middleware-mom-1.0/middleware-mom/lib/jackson-core-2.5.3.jar new file mode 100644 index 0000000..f3ae1ae Binary files /dev/null and b/target/lib/middleware-mom-1.0/middleware-mom/lib/jackson-core-2.5.3.jar differ diff --git a/target/lib/middleware-mom-1.0/middleware-mom/lib/javassist-3.19.0-GA.jar b/target/lib/middleware-mom-1.0/middleware-mom/lib/javassist-3.19.0-GA.jar new file mode 100644 index 0000000..7ac4264 Binary files /dev/null and b/target/lib/middleware-mom-1.0/middleware-mom/lib/javassist-3.19.0-GA.jar differ diff --git a/target/lib/middleware-mom-1.0/middleware-mom/lib/json-smart-2.0.jar b/target/lib/middleware-mom-1.0/middleware-mom/lib/json-smart-2.0.jar new file mode 100644 index 0000000..94409bb Binary files /dev/null and b/target/lib/middleware-mom-1.0/middleware-mom/lib/json-smart-2.0.jar differ diff --git a/target/lib/middleware-mom-1.0/middleware-mom/lib/kryo-3.0.2.jar b/target/lib/middleware-mom-1.0/middleware-mom/lib/kryo-3.0.2.jar new file mode 100644 index 0000000..4e8eee8 Binary files /dev/null and b/target/lib/middleware-mom-1.0/middleware-mom/lib/kryo-3.0.2.jar differ diff --git a/target/lib/middleware-mom-1.0/middleware-mom/lib/log4j-1.2.12.jar b/target/lib/middleware-mom-1.0/middleware-mom/lib/log4j-1.2.12.jar new file mode 100644 index 0000000..9b5a720 Binary files /dev/null and b/target/lib/middleware-mom-1.0/middleware-mom/lib/log4j-1.2.12.jar differ diff --git a/target/lib/middleware-mom-1.0/middleware-mom/lib/middleware-mom-1.0.jar b/target/lib/middleware-mom-1.0/middleware-mom/lib/middleware-mom-1.0.jar new file mode 100644 index 0000000..315a206 Binary files /dev/null and b/target/lib/middleware-mom-1.0/middleware-mom/lib/middleware-mom-1.0.jar differ diff --git a/target/lib/middleware-mom-1.0/middleware-mom/lib/minlog-1.3.0.jar b/target/lib/middleware-mom-1.0/middleware-mom/lib/minlog-1.3.0.jar new file mode 100644 index 0000000..ec67b63 Binary files /dev/null and b/target/lib/middleware-mom-1.0/middleware-mom/lib/minlog-1.3.0.jar differ diff --git a/target/lib/middleware-mom-1.0/middleware-mom/lib/netty-all-4.0.23.Final.jar b/target/lib/middleware-mom-1.0/middleware-mom/lib/netty-all-4.0.23.Final.jar new file mode 100644 index 0000000..0555a16 Binary files /dev/null and b/target/lib/middleware-mom-1.0/middleware-mom/lib/netty-all-4.0.23.Final.jar differ diff --git a/target/lib/middleware-mom-1.0/middleware-mom/lib/objenesis-2.1.jar b/target/lib/middleware-mom-1.0/middleware-mom/lib/objenesis-2.1.jar new file mode 100644 index 0000000..7c1e983 Binary files /dev/null and b/target/lib/middleware-mom-1.0/middleware-mom/lib/objenesis-2.1.jar differ diff --git a/target/lib/middleware-mom-1.0/middleware-mom/lib/reflectasm-1.10.1.jar b/target/lib/middleware-mom-1.0/middleware-mom/lib/reflectasm-1.10.1.jar new file mode 100644 index 0000000..ca81b39 Binary files /dev/null and b/target/lib/middleware-mom-1.0/middleware-mom/lib/reflectasm-1.10.1.jar differ diff --git a/target/lib/middleware-mom-1.0/middleware-mom/lib/slf4j-api-1.7.2.jar b/target/lib/middleware-mom-1.0/middleware-mom/lib/slf4j-api-1.7.2.jar new file mode 100644 index 0000000..1a88708 Binary files /dev/null and b/target/lib/middleware-mom-1.0/middleware-mom/lib/slf4j-api-1.7.2.jar differ diff --git a/target/lib/middleware-mom-1.0/middleware-mom/lib/slf4j-log4j12-1.7.2.jar b/target/lib/middleware-mom-1.0/middleware-mom/lib/slf4j-log4j12-1.7.2.jar new file mode 100644 index 0000000..5dfcf4b Binary files /dev/null and b/target/lib/middleware-mom-1.0/middleware-mom/lib/slf4j-log4j12-1.7.2.jar differ diff --git a/target/lib/test-classes/com/alibaba/middleware/race/mom/AppTest$1.class b/target/lib/test-classes/com/alibaba/middleware/race/mom/AppTest$1.class new file mode 100644 index 0000000..ab6b89a Binary files /dev/null and b/target/lib/test-classes/com/alibaba/middleware/race/mom/AppTest$1.class differ diff --git a/target/lib/test-classes/com/alibaba/middleware/race/mom/AppTest.class b/target/lib/test-classes/com/alibaba/middleware/race/mom/AppTest.class new file mode 100644 index 0000000..bd00c1b Binary files /dev/null and b/target/lib/test-classes/com/alibaba/middleware/race/mom/AppTest.class differ diff --git a/target/lib/test-classes/com/alibaba/middleware/race/mom/Group.class b/target/lib/test-classes/com/alibaba/middleware/race/mom/Group.class new file mode 100644 index 0000000..ad88663 Binary files /dev/null and b/target/lib/test-classes/com/alibaba/middleware/race/mom/Group.class differ diff --git a/target/lib/test-classes/com/alibaba/middleware/race/mom/User.class b/target/lib/test-classes/com/alibaba/middleware/race/mom/User.class new file mode 100644 index 0000000..f9d1af9 Binary files /dev/null and b/target/lib/test-classes/com/alibaba/middleware/race/mom/User.class differ diff --git a/target/lib/test-classes/com/alibaba/middleware/race/mom/fstjson.class b/target/lib/test-classes/com/alibaba/middleware/race/mom/fstjson.class new file mode 100644 index 0000000..9cc91a9 Binary files /dev/null and b/target/lib/test-classes/com/alibaba/middleware/race/mom/fstjson.class differ diff --git a/target/lib/test-classes/com/alibaba/middleware/race/mom/test$1.class b/target/lib/test-classes/com/alibaba/middleware/race/mom/test$1.class new file mode 100644 index 0000000..847424f Binary files /dev/null and b/target/lib/test-classes/com/alibaba/middleware/race/mom/test$1.class differ diff --git a/target/lib/test-classes/com/alibaba/middleware/race/mom/test.class b/target/lib/test-classes/com/alibaba/middleware/race/mom/test.class new file mode 100644 index 0000000..c3d30a1 Binary files /dev/null and b/target/lib/test-classes/com/alibaba/middleware/race/mom/test.class differ diff --git a/target/maven-archiver/pom.properties b/target/maven-archiver/pom.properties new file mode 100644 index 0000000..fbaae2d --- /dev/null +++ b/target/maven-archiver/pom.properties @@ -0,0 +1,5 @@ +#Generated by Maven +#Thu Jul 23 15:06:51 CST 2015 +version=1.0 +groupId=com.alibaba.race +artifactId=middleware-mom diff --git a/target/middleware-mom-1.0.jar b/target/middleware-mom-1.0.jar new file mode 100644 index 0000000..63069fa Binary files /dev/null and b/target/middleware-mom-1.0.jar differ diff --git a/target/surefire-reports/TEST-com.alibaba.middleware.race.mom.AppTest.xml b/target/surefire-reports/TEST-com.alibaba.middleware.race.mom.AppTest.xml new file mode 100644 index 0000000..be7c3de --- /dev/null +++ b/target/surefire-reports/TEST-com.alibaba.middleware.race.mom.AppTest.xml @@ -0,0 +1,64 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/target/surefire-reports/com.alibaba.middleware.race.mom.AppTest.txt b/target/surefire-reports/com.alibaba.middleware.race.mom.AppTest.txt new file mode 100644 index 0000000..e5d7765 --- /dev/null +++ b/target/surefire-reports/com.alibaba.middleware.race.mom.AppTest.txt @@ -0,0 +1,4 @@ +------------------------------------------------------------------------------- +Test set: com.alibaba.middleware.race.mom.AppTest +------------------------------------------------------------------------------- +Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.055 sec diff --git a/target/test-classes/com/alibaba/middleware/race/mom/AppTest$1.class b/target/test-classes/com/alibaba/middleware/race/mom/AppTest$1.class new file mode 100644 index 0000000..2c8cfe5 Binary files /dev/null and b/target/test-classes/com/alibaba/middleware/race/mom/AppTest$1.class differ diff --git a/target/test-classes/com/alibaba/middleware/race/mom/AppTest.class b/target/test-classes/com/alibaba/middleware/race/mom/AppTest.class new file mode 100644 index 0000000..e8df057 Binary files /dev/null and b/target/test-classes/com/alibaba/middleware/race/mom/AppTest.class differ diff --git a/target/test-classes/com/alibaba/middleware/race/mom/Group.class b/target/test-classes/com/alibaba/middleware/race/mom/Group.class new file mode 100644 index 0000000..4557917 Binary files /dev/null and b/target/test-classes/com/alibaba/middleware/race/mom/Group.class differ diff --git a/target/test-classes/com/alibaba/middleware/race/mom/User.class b/target/test-classes/com/alibaba/middleware/race/mom/User.class new file mode 100644 index 0000000..3eee391 Binary files /dev/null and b/target/test-classes/com/alibaba/middleware/race/mom/User.class differ diff --git a/target/test-classes/com/alibaba/middleware/race/mom/fstjson.class b/target/test-classes/com/alibaba/middleware/race/mom/fstjson.class new file mode 100644 index 0000000..570147b Binary files /dev/null and b/target/test-classes/com/alibaba/middleware/race/mom/fstjson.class differ diff --git a/target/test-classes/com/alibaba/middleware/race/mom/test$1.class b/target/test-classes/com/alibaba/middleware/race/mom/test$1.class new file mode 100644 index 0000000..7b77184 Binary files /dev/null and b/target/test-classes/com/alibaba/middleware/race/mom/test$1.class differ diff --git a/target/test-classes/com/alibaba/middleware/race/mom/test.class b/target/test-classes/com/alibaba/middleware/race/mom/test.class new file mode 100644 index 0000000..005b424 Binary files /dev/null and b/target/test-classes/com/alibaba/middleware/race/mom/test.class differ