diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AbstractAmqpExchange.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AbstractAmqpExchange.java index 3f2fe1a1..f984e81c 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AbstractAmqpExchange.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AbstractAmqpExchange.java @@ -18,12 +18,15 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import lombok.Getter; /** * Base class of AMQP exchange. */ public abstract class AbstractAmqpExchange implements AmqpExchange { + @Getter + protected final Map properties; protected final String exchangeName; protected final AmqpExchange.Type exchangeType; protected Set queues; @@ -37,7 +40,8 @@ public abstract class AbstractAmqpExchange implements AmqpExchange { protected AbstractAmqpExchange(String exchangeName, AmqpExchange.Type exchangeType, Set queues, boolean durable, boolean autoDelete, boolean internal, - Map arguments) { + Map arguments, Map properties) { + this.properties = properties; this.exchangeName = exchangeName; this.exchangeType = exchangeType; this.queues = queues; diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AbstractAmqpQueue.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AbstractAmqpQueue.java index d59a6c25..243fa994 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AbstractAmqpQueue.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AbstractAmqpQueue.java @@ -14,10 +14,12 @@ package io.streamnative.pulsar.handlers.amqp; import java.util.Collection; +import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import lombok.Getter; /** * Base class for AMQP queue. @@ -31,6 +33,9 @@ public abstract class AbstractAmqpQueue implements AmqpQueue { protected boolean exclusive; protected boolean autoDelete; protected final Map routers = new ConcurrentHashMap<>(); + protected final Map arguments = new HashMap<>(); + @Getter + protected Map properties; protected AbstractAmqpQueue(String queueName, boolean durable, long connectionId) { this.queueName = queueName; @@ -42,12 +47,13 @@ protected AbstractAmqpQueue(String queueName, boolean durable, long connectionId protected AbstractAmqpQueue(String queueName, boolean durable, long connectionId, - boolean exclusive, boolean autoDelete) { + boolean exclusive, boolean autoDelete, Map properties) { this.queueName = queueName; this.durable = durable; this.connectionId = connectionId; this.exclusive = exclusive; this.autoDelete = autoDelete; + this.properties = properties; } @Override @@ -60,6 +66,20 @@ public boolean getDurable() { return durable; } + @Override + public boolean getExclusive() { + return exclusive; + } + @Override + public boolean getAutoDelete() { + return autoDelete; + } + + @Override + public Map getArguments() { + return arguments; + } + @Override public AmqpMessageRouter getRouter(String exchangeName) { return routers.get(exchangeName); @@ -133,4 +153,8 @@ public boolean isAutoDelete() { return autoDelete; } + @Override + public void close() { + + } } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpBrokerService.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpBrokerService.java index 19cfac8e..4749f5d5 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpBrokerService.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpBrokerService.java @@ -16,11 +16,16 @@ import io.netty.util.concurrent.DefaultThreadFactory; import io.streamnative.pulsar.handlers.amqp.admin.AmqpAdmin; +import io.streamnative.pulsar.handlers.amqp.common.exception.AoPServiceRuntimeException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import lombok.Getter; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.SizeUnit; /** * AMQP broker related. @@ -43,16 +48,32 @@ public class AmqpBrokerService { @Getter private AmqpAdmin amqpAdmin; + @Getter + private final PulsarClient pulsarClient; + public AmqpBrokerService(PulsarService pulsarService, AmqpServiceConfiguration config) { + try { + this.pulsarClient = PulsarClient.builder() + .serviceUrl(config.getAmqpServerAddress()) + .ioThreads(config.getNumIOThreads()) + .listenerThreads(config.getNumIOThreads()) + .memoryLimit(0, SizeUnit.BYTES) + .statsInterval(0, TimeUnit.MILLISECONDS) + .connectionMaxIdleSeconds(-1) + .build(); + } catch (PulsarClientException e) { + throw new AoPServiceRuntimeException(e); + } + String clusterName = pulsarService.getBrokerService().getPulsar().getConfiguration().getClusterName(); + this.amqpAdmin = new AmqpAdmin(config.getAdvertisedAddress(), config.getAmqpAdminPort()); this.pulsarService = pulsarService; this.amqpTopicManager = new AmqpTopicManager(pulsarService); this.exchangeContainer = new ExchangeContainer(amqpTopicManager, pulsarService, - initRouteExecutor(config), config); + initRouteExecutor(config), config, amqpAdmin, pulsarClient); this.queueContainer = new QueueContainer(amqpTopicManager, pulsarService, exchangeContainer, config); this.exchangeService = new ExchangeServiceImpl(exchangeContainer); - this.queueService = new QueueServiceImpl(exchangeContainer, queueContainer); - this.connectionContainer = new ConnectionContainer(pulsarService, exchangeContainer, queueContainer); - this.amqpAdmin = new AmqpAdmin("localhost", config.getAmqpAdminPort()); + this.queueService = new QueueServiceImpl(exchangeContainer, queueContainer, amqpTopicManager); + this.connectionContainer = new ConnectionContainer(pulsarService, exchangeContainer, queueContainer, config); } private ExecutorService initRouteExecutor(AmqpServiceConfiguration config) { diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpChannel.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpChannel.java index a5c28612..cc7ab92a 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpChannel.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpChannel.java @@ -216,7 +216,7 @@ public void receiveExchangeBound(AMQShortString exchange, AMQShortString routing } exchangeService.exchangeBound( - connection.getNamespaceName(), exchange.toString(), routingKey.toString(), queueName.toString()) + connection.getNamespaceName(), exchange.toString(), routingKey.toString(), queueName.toString()) .thenAccept(replyCode -> { String replyText = null; switch (replyCode) { @@ -253,18 +253,18 @@ public void receiveQueueDeclare(AMQShortString queue, boolean passive, boolean d channelId, queue, passive, durable, exclusive, autoDelete, nowait, arguments); } queueService.queueDeclare(connection.getNamespaceName(), queue.toString(), passive, durable, exclusive, - autoDelete, nowait, FieldTable.convertToMap(arguments), connection.getConnectionId()) + autoDelete, nowait, FieldTable.convertToMap(arguments), connection.getConnectionId()) .thenAccept(amqpQueue -> { - setDefaultQueue(amqpQueue); - MethodRegistry methodRegistry = connection.getMethodRegistry(); - QueueDeclareOkBody responseBody = methodRegistry.createQueueDeclareOkBody( - AMQShortString.createAMQShortString(amqpQueue.getName()), 0, 0); - connection.writeFrame(responseBody.generateFrame(channelId)); - }).exceptionally(t -> { - log.error("Failed to declare queue {} in vhost {}", queue, connection.getNamespaceName(), t); - handleAoPException(t); - return null; - }); + setDefaultQueue(amqpQueue); + MethodRegistry methodRegistry = connection.getMethodRegistry(); + QueueDeclareOkBody responseBody = methodRegistry.createQueueDeclareOkBody( + AMQShortString.createAMQShortString(amqpQueue.getName()), 0, 0); + connection.writeFrame(responseBody.generateFrame(channelId)); + }).exceptionally(t -> { + log.error("Failed to declare queue {} in vhost {}", queue, connection.getNamespaceName(), t); + handleAoPException(t); + return null; + }); } @Override @@ -361,7 +361,7 @@ public void receiveQueueUnbind(AMQShortString queue, AMQShortString exchange, AM public void receiveBasicQos(long prefetchSize, int prefetchCount, boolean global) { if (log.isDebugEnabled()) { log.debug("RECV[{}] BasicQos[prefetchSize: {} prefetchCount: {} global: {}]", - channelId, prefetchSize, prefetchCount, global); + channelId, prefetchSize, prefetchCount, global); } if (prefetchSize > 0) { closeChannel(ErrorCodes.NOT_IMPLEMENTED, "prefetchSize not supported "); @@ -412,7 +412,7 @@ protected String getConsumerTag(AMQShortString consumerTag) { } private synchronized void subscribe(String consumerTag, String queueName, Topic topic, - boolean ack, boolean exclusive, boolean nowait) { + boolean ack, boolean exclusive, boolean nowait) { CompletableFuture future = new CompletableFuture<>(); future.whenComplete((ignored, e) -> { @@ -432,28 +432,28 @@ private synchronized void subscribe(String consumerTag, String queueName, Topic CompletableFuture subscriptionFuture = topic.createSubscription( defaultSubscription, CommandSubscribe.InitialPosition.Earliest, false, null); subscriptionFuture.thenAccept(subscription -> { - AmqpConsumer consumer = new AmqpConsumer(queueContainer, subscription, - exclusive ? CommandSubscribe.SubType.Exclusive : CommandSubscribe.SubType.Shared, - topic.getName(), CONSUMER_ID.incrementAndGet(), 0, - consumerTag, true, connection.getServerCnx(), "", null, - false, MessageId.latest, - null, this, consumerTag, queueName, ack); - subscription.addConsumer(consumer).thenAccept(__ -> { - consumer.handleFlow(DEFAULT_CONSUMER_PERMIT); - tag2ConsumersMap.put(consumerTag, consumer); - - if (!nowait) { - MethodRegistry methodRegistry = connection.getMethodRegistry(); - AMQMethodBody responseBody = methodRegistry. - createBasicConsumeOkBody(AMQShortString. - createAMQShortString(consumer.getConsumerTag())); - connection.writeFrame(responseBody.generateFrame(channelId)); - } - future.complete(null); - }).exceptionally(t -> { - future.completeExceptionally(t); - return null; - }); + AmqpConsumer consumer = new AmqpConsumer(queueContainer, subscription, + exclusive ? CommandSubscribe.SubType.Exclusive : CommandSubscribe.SubType.Shared, + topic.getName(), CONSUMER_ID.incrementAndGet(), 0, + consumerTag, true, connection.getServerCnx(), "", null, + false, MessageId.latest, + null, this, consumerTag, queueName, ack); + subscription.addConsumer(consumer).thenAccept(__ -> { + consumer.handleFlow(DEFAULT_CONSUMER_PERMIT); + tag2ConsumersMap.put(consumerTag, consumer); + + if (!nowait) { + MethodRegistry methodRegistry = connection.getMethodRegistry(); + AMQMethodBody responseBody = methodRegistry. + createBasicConsumeOkBody(AMQShortString. + createAMQShortString(consumer.getConsumerTag())); + connection.writeFrame(responseBody.generateFrame(channelId)); + } + future.complete(null); + }).exceptionally(t -> { + future.completeExceptionally(t); + return null; + }); }).exceptionally(t -> { future.completeExceptionally(t); return null; @@ -740,17 +740,19 @@ public boolean ignoreAllButCloseOk() { public void receiveBasicNack(long deliveryTag, boolean multiple, boolean requeue) { if (log.isDebugEnabled()) { log.debug("RECV[ {} ] BasicNAck[deliveryTag: {} multiple: {} requeue: {}]", - channelId, deliveryTag, multiple, requeue); + channelId, deliveryTag, multiple, requeue); } messageNAck(deliveryTag, multiple, requeue); } public void messageNAck(long deliveryTag, boolean multiple, boolean requeue) { Collection ackedMessages = - unacknowledgedMessageMap.acknowledge(deliveryTag, multiple); + unacknowledgedMessageMap.acknowledge(deliveryTag, multiple); if (!ackedMessages.isEmpty()) { if (requeue) { requeue(ackedMessages); + } else { + discardMessage(ackedMessages); } } else { closeChannel(ErrorCodes.IN_USE, "deliveryTag not found"); @@ -760,10 +762,21 @@ public void messageNAck(long deliveryTag, boolean multiple, boolean requeue) { } } + private void discardMessage(Collection messages) { + Map> positionMap = new HashMap<>(); + messages.forEach(association -> { + UnacknowledgedMessageMap.MessageProcessor consumer = association.getConsumer(); + List positions = positionMap.computeIfAbsent(consumer, + list -> new ArrayList<>()); + positions.add((PositionImpl) association.getPosition()); + }); + positionMap.forEach(UnacknowledgedMessageMap.MessageProcessor::discardMessage); + } + @Override public void receiveBasicRecover(boolean requeue, boolean sync) { Collection ackedMessages = - unacknowledgedMessageMap.acknowledgeAll(); + unacknowledgedMessageMap.acknowledgeAll(); if (!ackedMessages.isEmpty()) { requeue(ackedMessages); } @@ -782,7 +795,7 @@ private void requeue(Collection { UnacknowledgedMessageMap.MessageProcessor consumer = association.getConsumer(); List positions = positionMap.computeIfAbsent(consumer, - list -> new ArrayList<>()); + list -> new ArrayList<>()); positions.add((PositionImpl) association.getPosition()); }); positionMap.entrySet().stream().forEach(entry -> { @@ -806,7 +819,8 @@ private void messageAck(long deliveryTag, boolean multiple) { entry.getConsumer().messageAck(entry.getPosition()); }); } else { - closeChannel(ErrorCodes.IN_USE, "deliveryTag not found"); + // TODO + // closeChannel(ErrorCodes.IN_USE, "deliveryTag not found"); } if (creditManager.hasCredit() && isBlockedOnCredit()) { unBlockedOnCredit(); @@ -904,7 +918,7 @@ public void closeChannel(int cause, final String message) { connection.closeChannelAndWriteFrame(this, cause, message); } - public long getNextDeliveryTag() { + public synchronized long getNextDeliveryTag() { return ++deliveryTag; } @@ -1015,16 +1029,16 @@ public AmqpFlowCreditManager getCreditManager() { protected void handleAoPException(Throwable t) { Throwable cause = FutureUtil.unwrapCompletionException(t); - if (!(cause instanceof AoPException)) { + if (!(cause instanceof AoPException exception)) { connection.sendConnectionClose(INTERNAL_ERROR, t.getMessage(), channelId); return; } - AoPException exception = (AoPException) cause; - if (exception.isCloseChannel()) { - closeChannel(exception.getErrorCode(), exception.getMessage()); - } if (exception.isCloseConnection()) { connection.sendConnectionClose(exception.getErrorCode(), exception.getMessage(), channelId); + } else if (exception.isCloseChannel()) { + closeChannel(exception.getErrorCode(), exception.getMessage()); + } else { + connection.sendConnectionClose(INTERNAL_ERROR, exception.getMessage(), channelId); } } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpConnection.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpConnection.java index 69f3c878..9a42ca58 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpConnection.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpConnection.java @@ -15,7 +15,6 @@ import static com.google.common.base.Preconditions.checkState; import static java.nio.charset.StandardCharsets.US_ASCII; - import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.ByteBuf; import io.netty.channel.Channel; @@ -28,7 +27,12 @@ import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; +import java.util.StringTokenizer; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -36,16 +40,16 @@ import lombok.extern.log4j.Log4j2; import org.apache.bookkeeper.util.collections.ConcurrentLongLongHashMap; import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.authentication.AuthenticationProvider; import org.apache.pulsar.broker.authentication.AuthenticationState; -import org.apache.pulsar.broker.namespace.LookupOptions; import org.apache.pulsar.broker.service.ServerCnx; +import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.common.api.AuthData; import org.apache.pulsar.common.naming.NamespaceName; -import org.apache.pulsar.common.naming.TopicDomain; -import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap; import org.apache.qpid.server.bytebuffer.QpidByteBuffer; +import org.apache.qpid.server.common.ServerPropertyNames; import org.apache.qpid.server.protocol.ErrorCodes; import org.apache.qpid.server.protocol.ProtocolVersion; import org.apache.qpid.server.protocol.v0_8.AMQDecoder; @@ -59,6 +63,7 @@ import org.apache.qpid.server.protocol.v0_8.transport.ConnectionCloseOkBody; import org.apache.qpid.server.protocol.v0_8.transport.ConnectionTuneBody; import org.apache.qpid.server.protocol.v0_8.transport.HeartbeatBody; +import org.apache.qpid.server.protocol.v0_8.transport.MessagePublishInfo; import org.apache.qpid.server.protocol.v0_8.transport.MethodRegistry; import org.apache.qpid.server.protocol.v0_8.transport.ProtocolInitiation; import org.apache.qpid.server.protocol.v0_8.transport.ServerChannelMethodProcessor; @@ -86,6 +91,7 @@ enum ConnectionState { private static final AtomicLong ID_GENERATOR = new AtomicLong(0); private long connectionId; + @Getter private final ConcurrentLongHashMap channels; private final ConcurrentLongLongHashMap closingChannelsList = new ConcurrentLongLongHashMap(); @Getter @@ -109,10 +115,19 @@ enum ConnectionState { @Getter private AmqpBrokerService amqpBrokerService; private AuthenticationState authenticationState; + @Getter + private String clientIp; + @Getter + private long connectedAt; + + final Map>> producerMap; + final Map publishInfoMap; + private String tenant; public AmqpConnection(AmqpServiceConfiguration amqpConfig, AmqpBrokerService amqpBrokerService) { super(amqpBrokerService.getPulsarService(), amqpConfig); + this.tenant = amqpConfig.getAmqpTenant(); this.connectionId = ID_GENERATOR.incrementAndGet(); this.channels = new ConcurrentLongHashMap<>(); this.protocolVersion = ProtocolVersion.v0_91; @@ -124,6 +139,8 @@ public AmqpConnection(AmqpServiceConfiguration amqpConfig, this.heartBeat = amqpConfig.getAmqpHeartBeat(); this.amqpOutputConverter = new AmqpOutputConverter(this); this.amqpBrokerService = amqpBrokerService; + this.producerMap = new ConcurrentHashMap<>(); + this.publishInfoMap = new ConcurrentHashMap<>(); } @@ -141,6 +158,7 @@ public void channelActive(ChannelHandlerContext ctx) throws Exception { public void channelInactive(ChannelHandlerContext ctx) throws Exception { super.channelInactive(ctx); completeAndCloseAllChannels(); + closeAllProducers(); amqpBrokerService.getConnectionContainer().removeConnection(namespaceName, this); this.brokerDecoder.close(); } @@ -167,7 +185,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - log.error("[{}] Got exception: {}", remoteAddress, cause.getMessage(), cause); + log.error("[{}] Got exception: {}", clientIp, cause.getMessage(), cause); close(); } @@ -176,6 +194,7 @@ protected void close() { if (isActive.getAndSet(false)) { log.info("close netty channel {}", ctx.channel()); ctx.close(); + bufferSender.close(); } } @@ -186,6 +205,7 @@ public void receiveConnectionStartOk(FieldTable clientProperties, AMQShortString log.debug("RECV ConnectionStartOk[clientProperties: {}, mechanism: {}, locale: {}]", clientProperties, mechanism, locale); } + this.clientIp = ctx.channel().remoteAddress().toString(); assertState(ConnectionState.AWAIT_START_OK); // TODO clientProperties @@ -276,7 +296,7 @@ public void receiveConnectionSecureOk(byte[] response) { public void receiveConnectionTuneOk(int channelMax, long frameMax, int heartbeat) { if (log.isDebugEnabled()) { log.debug("RECV ConnectionTuneOk[ channelMax: {} frameMax: {} heartbeat: {} ]", - channelMax, frameMax, heartbeat); + channelMax, frameMax, heartbeat); } assertState(ConnectionState.AWAIT_TUNE_OK); @@ -293,22 +313,22 @@ public void receiveConnectionTuneOk(int channelMax, long frameMax, int heartbeat if (frameMax > (long) brokerFrameMax) { sendConnectionClose(ErrorCodes.SYNTAX_ERROR, - "Attempt to set max frame size to " + frameMax - + " greater than the broker will allow: " - + brokerFrameMax, 0); + "Attempt to set max frame size to " + frameMax + + " greater than the broker will allow: " + + brokerFrameMax, 0); } else if (frameMax > 0 && frameMax < AMQDecoder.FRAME_MIN_SIZE) { sendConnectionClose(ErrorCodes.SYNTAX_ERROR, - "Attempt to set max frame size to " + frameMax - + " which is smaller than the specification defined minimum: " - + AMQFrame.getFrameOverhead(), 0); + "Attempt to set max frame size to " + frameMax + + " which is smaller than the specification defined minimum: " + + AMQFrame.getFrameOverhead(), 0); } else { int calculatedFrameMax = frameMax == 0 ? brokerFrameMax : (int) frameMax; setMaxFrameSize(calculatedFrameMax); //0 means no implied limit, except that forced by protocol limitations (0xFFFF) int value = ((channelMax == 0) || (channelMax > 0xFFFF)) - ? 0xFFFF - : channelMax; + ? 0xFFFF + : channelMax; maxChannels = value; } state = ConnectionState.AWAIT_OPEN; @@ -324,50 +344,38 @@ public void receiveConnectionOpen(AMQShortString virtualHost, AMQShortString cap assertState(ConnectionState.AWAIT_OPEN); - boolean isDefaultNamespace = false; String virtualHostStr = AMQShortString.toString(virtualHost); - if ((virtualHostStr != null) && virtualHostStr.charAt(0) == '/') { - virtualHostStr = virtualHostStr.substring(1); - if (StringUtils.isEmpty(virtualHostStr)){ - virtualHostStr = DEFAULT_NAMESPACE; - isDefaultNamespace = true; - } + Pair pair; + if (virtualHostStr == null || (pair = validateVirtualHost(virtualHostStr)) == null){ + sendConnectionClose(ErrorCodes.NOT_ALLOWED, String.format( + "The virtualHost [%s] configuration is incorrect. For example: tenant/namespace or namespace", + virtualHostStr), 0); + return; } - NamespaceName namespaceName = NamespaceName.get(amqpConfig.getAmqpTenant(), virtualHostStr); - if (isDefaultNamespace) { - // avoid the namespace public/default is not owned in standalone mode - TopicName topic = TopicName.get(TopicDomain.persistent.value(), - namespaceName, "__lookup__"); - LookupOptions lookupOptions = LookupOptions.builder().authoritative(true).build(); - getPulsarService().getNamespaceService().getBrokerServiceUrlAsync(topic, lookupOptions); - } - // Policies policies = getPolicies(namespaceName); -// if (policies != null) { + NamespaceName namespaceName = NamespaceName.get(pair.getLeft(), pair.getRight()); this.namespaceName = namespaceName; MethodRegistry methodRegistry = getMethodRegistry(); AMQMethodBody responseBody = methodRegistry.createConnectionOpenOkBody(virtualHost); writeFrame(responseBody.generateFrame(0)); state = ConnectionState.OPEN; + connectedAt = System.currentTimeMillis(); amqpBrokerService.getConnectionContainer().addConnection(namespaceName, this); -// } else { -// sendConnectionClose(ErrorCodes.NOT_FOUND, -// "Unknown virtual host: '" + virtualHostStr + "'", 0); -// } } @Override public void receiveConnectionClose(int replyCode, AMQShortString replyText, - int classId, int methodId) { + int classId, int methodId) { if (log.isDebugEnabled()) { log.debug("RECV ConnectionClose[ replyCode: {} replyText: {} classId: {} methodId: {} ]", - replyCode, replyText, classId, methodId); + replyCode, replyText, classId, methodId); } try { if (orderlyClose.compareAndSet(false, true)) { completeAndCloseAllChannels(); + closeAllProducers(); } MethodRegistry methodRegistry = getMethodRegistry(); @@ -398,6 +406,7 @@ private void sendConnectionClose(int channelId, AMQFrame frame) { try { markChannelAwaitingCloseOk(channelId); completeAndCloseAllChannels(); + closeAllProducers(); } finally { writeFrame(frame); } @@ -414,14 +423,14 @@ public void receiveChannelOpen(int channelId) { if (this.namespaceName == null) { sendConnectionClose(ErrorCodes.COMMAND_INVALID, - "Virtualhost has not yet been set. ConnectionOpen has not been called.", channelId); + "Virtualhost has not yet been set. ConnectionOpen has not been called.", channelId); } else if (channels.get(channelId) != null || channelAwaitingClosure(channelId)) { sendConnectionClose(ErrorCodes.CHANNEL_ERROR, "Channel " + channelId + " already exists", channelId); } else if (channelId > maxChannels) { sendConnectionClose(ErrorCodes.CHANNEL_ERROR, - "Channel " + channelId + " cannot be created as the max allowed channel id is " - + maxChannels, - channelId); + "Channel " + channelId + " cannot be created as the max allowed channel id is " + + maxChannels, + channelId); } else { log.debug("Connecting to: {}", namespaceName.getLocalName()); final AmqpChannel channel; @@ -467,7 +476,11 @@ public void receiveProtocolHeader(ProtocolInitiation pi) { AMQMethodBody responseBody = this.methodRegistry.createConnectionStartBody( (short) protocolVersion.getMajorVersion(), (short) pv.getActualMinorVersion(), - null, + FieldTable.convertToFieldTable(new HashMap<>(2) { + { + put(ServerPropertyNames.VERSION, AopVersion.getVersion()); + } + }), // TODO temporary modification "PLAIN token".getBytes(US_ASCII), "en_US".getBytes(US_ASCII)); @@ -491,22 +504,22 @@ public ServerChannelMethodProcessor getChannelMethodProcessor(int channelId) { ServerChannelMethodProcessor channelMethodProcessor = getChannel(channelId); if (channelMethodProcessor == null) { channelMethodProcessor = - (ServerChannelMethodProcessor) Proxy.newProxyInstance(ServerMethodDispatcher.class.getClassLoader(), - new Class[] {ServerChannelMethodProcessor.class}, new InvocationHandler() { - @Override - public Object invoke(final Object proxy, final Method method, final Object[] args) - throws Throwable { - if (method.getName().equals("receiveChannelCloseOk") && channelAwaitingClosure(channelId)) { - closeChannelOk(channelId); - } else if (method.getName().startsWith("receive")) { - sendConnectionClose(ErrorCodes.CHANNEL_ERROR, - "Unknown channel id: " + channelId, channelId); - } else if (method.getName().equals("ignoreAllButCloseOk")) { - return channelAwaitingClosure(channelId); - } - return null; - } - }); + (ServerChannelMethodProcessor) Proxy.newProxyInstance(ServerMethodDispatcher.class.getClassLoader(), + new Class[] {ServerChannelMethodProcessor.class}, new InvocationHandler() { + @Override + public Object invoke(final Object proxy, final Method method, final Object[] args) + throws Throwable { + if (method.getName().equals("receiveChannelCloseOk") && channelAwaitingClosure(channelId)) { + closeChannelOk(channelId); + } else if (method.getName().startsWith("receive")) { + sendConnectionClose(ErrorCodes.CHANNEL_ERROR, + "Unknown channel id: " + channelId, channelId); + } else if (method.getName().equals("ignoreAllButCloseOk")) { + return channelAwaitingClosure(channelId); + } + return null; + } + }); } return channelMethodProcessor; } @@ -525,10 +538,23 @@ void assertState(final ConnectionState requiredState) { } } + private Pair validateVirtualHost(String virtualHostStr) { + String virtualHost = virtualHostStr.trim(); + if ("/".equals(virtualHost)) { + return Pair.of(this.tenant, AmqpConnection.DEFAULT_NAMESPACE); + } + StringTokenizer tokenizer = new StringTokenizer(virtualHost, "/", false); + return switch (tokenizer.countTokens()) { + case 1 -> Pair.of(this.tenant, tokenizer.nextToken()); + case 2 -> Pair.of(tokenizer.nextToken(), tokenizer.nextToken()); + default -> null; + }; + } + public boolean channelAwaitingClosure(int channelId) { return ignoreAllButCloseOk() || (!closingChannelsList.isEmpty() - && closingChannelsList.containsKey(channelId)); + && closingChannelsList.containsKey(channelId)); } public void completeAndCloseAllChannels() { @@ -550,7 +576,7 @@ private void receivedCompleteAllChannels() { exception = exceptionForThisChannel; } log.error("error informing channel that receiving is complete. Channel: " + channel, - exceptionForThisChannel); + exceptionForThisChannel); } } @@ -593,7 +619,7 @@ public void setHeartBeat(int heartBeat) { public void initHeartBeatHandler(long writerIdle, long readerIdle) { this.ctx.pipeline().addFirst("idleStateHandler", new IdleStateHandler(readerIdle, writerIdle, 0, - TimeUnit.MILLISECONDS)); + TimeUnit.MILLISECONDS)); this.ctx.pipeline().addLast("connectionIdleHandler", new ConnectionIdleHandler()); } @@ -605,10 +631,10 @@ class ConnectionIdleHandler extends ChannelDuplexHandler { IdleStateEvent event = (IdleStateEvent) evt; if (event.state().equals(IdleState.READER_IDLE)) { log.error("heartbeat timeout close remoteSocketAddress [{}]", - AmqpConnection.this.remoteAddress.toString()); + AmqpConnection.this.remoteAddress.toString()); AmqpConnection.this.close(); } else if (event.state().equals(IdleState.WRITER_IDLE)) { - log.warn("heartbeat write idle [{}]", AmqpConnection.this.remoteAddress.toString()); + //log.warn("heartbeat write idle [{}]", AmqpConnection.this.remoteAddress.toString()); writeFrame(HeartbeatBody.FRAME); } } @@ -661,10 +687,10 @@ public void closeChannel(AmqpChannel channel) { public void closeChannelAndWriteFrame(AmqpChannel channel, int cause, String message) { writeFrame(new AMQFrame(channel.getChannelId(), - getMethodRegistry().createChannelCloseBody(cause, - AMQShortString.validValueOf(message), - currentClassId, - currentMethodId))); + getMethodRegistry().createChannelCloseBody(cause, + AMQShortString.validValueOf(message), + currentClassId, + currentMethodId))); closeChannel(channel, true); } @@ -680,6 +706,13 @@ void closeChannel(AmqpChannel channel, boolean mark) { } } + private void closeAllProducers() { + for (CompletableFuture> producer : producerMap.values()) { + producer.thenApply(Producer::closeAsync); + } + producerMap.clear(); + } + private void closeAllChannels() { RuntimeException exception = null; try { @@ -691,7 +724,7 @@ private void closeAllChannels() { exception = exceptionForThisChannel; } log.error("error informing channel that receiving is complete. Channel: " + channel, - exceptionForThisChannel); + exceptionForThisChannel); } } if (exception != null) { diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpConsumer.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpConsumer.java index 370d2ad7..66d7b08b 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpConsumer.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpConsumer.java @@ -76,13 +76,13 @@ public class AmqpConsumer extends Consumer implements UnacknowledgedMessageMap.M private final int maxPermits = 1000; public AmqpConsumer(QueueContainer queueContainer, Subscription subscription, - CommandSubscribe.SubType subType, String topicName, long consumerId, - int priorityLevel, String consumerName, boolean isDurable, ServerCnx cnx, - String appId, Map metadata, boolean readCompacted, MessageId messageId, - KeySharedMeta keySharedMeta, AmqpChannel channel, String consumerTag, String queueName, - boolean autoAck) { + CommandSubscribe.SubType subType, String topicName, long consumerId, + int priorityLevel, String consumerName, boolean isDurable, ServerCnx cnx, + String appId, Map metadata, boolean readCompacted, MessageId messageId, + KeySharedMeta keySharedMeta, AmqpChannel channel, String consumerTag, String queueName, + boolean autoAck) { super(subscription, subType, topicName, consumerId, priorityLevel, consumerName, isDurable, - cnx, appId, metadata, readCompacted, keySharedMeta, messageId, Commands.DEFAULT_CONSUMER_EPOCH); + cnx, appId, metadata, readCompacted, keySharedMeta, messageId, Commands.DEFAULT_CONSUMER_EPOCH); this.channel = channel; this.queueContainer = queueContainer; this.autoAck = autoAck; @@ -101,8 +101,8 @@ public Future sendMessages(final List entries, EntryBatch @Override public Future sendMessages(final List entries, EntryBatchSizes batchSizes, - EntryBatchIndexesAcks batchIndexesAcks, int totalMessages, long totalBytes, long totalChunkedMessages, - RedeliveryTracker redeliveryTracker, long epoch) { + EntryBatchIndexesAcks batchIndexesAcks, int totalMessages, long totalBytes, long totalChunkedMessages, + RedeliveryTracker redeliveryTracker, long epoch) { ChannelPromise writePromise = this.channel.getConnection().getCtx().newPromise(); if (entries.isEmpty() || totalMessages == 0) { if (log.isDebugEnabled()) { @@ -153,49 +153,50 @@ private CompletableFuture sendMessage(Entry index) { } asyncGetQueue() .thenCompose(amqpQueue -> amqpQueue.readEntryAsync( - indexMessage.getExchangeName(), indexMessage.getLedgerId(), indexMessage.getEntryId()) - .thenAccept(msg -> { - try { - long deliveryTag = channel.getNextDeliveryTag(); + indexMessage.getExchangeName(), indexMessage.getLedgerId(), indexMessage.getEntryId()) + .thenAccept(msg -> { + try { + long deliveryTag = channel.getNextDeliveryTag(); - addUnAckMessages(indexMessage.getExchangeName(), (PositionImpl) index.getPosition(), - (PositionImpl) msg.getPosition()); - if (!autoAck) { - channel.getUnacknowledgedMessageMap().add(deliveryTag, - index.getPosition(), this, msg.getLength()); - } + addUnAckMessages(indexMessage.getExchangeName(), (PositionImpl) index.getPosition(), + (PositionImpl) msg.getPosition()); + if (!autoAck) { + channel.getUnacknowledgedMessageMap().add(deliveryTag, + index.getPosition(), this, msg.getLength()); + } - try { - boolean isRedelivery = getRedeliveryTracker().getRedeliveryCount( - index.getPosition().getLedgerId(), index.getPosition().getEntryId()) > 0; - channel.getConnection().getAmqpOutputConverter().writeDeliver( - MessageConvertUtils.entryToAmqpBody(msg), - channel.getChannelId(), - isRedelivery, - deliveryTag, - AMQShortString.createAMQShortString(consumerTag)); - sendFuture.complete(null); - } catch (Exception e) { - log.error("[{}-{}] Failed to send message to consumer.", queueName, consumerTag, e); - sendFuture.completeExceptionally(e); - return; - } finally { - msg.release(); - } + try { + boolean isRedelivery = getRedeliveryTracker().getRedeliveryCount( + index.getPosition().getLedgerId(), + index.getPosition().getEntryId()) > 0; + channel.getConnection().getAmqpOutputConverter().writeDeliver( + MessageConvertUtils.entryToAmqpBody(msg), + channel.getChannelId(), + isRedelivery, + deliveryTag, + AMQShortString.createAMQShortString(consumerTag)); + sendFuture.complete(null); + } catch (Exception e) { + log.error("[{}-{}] Failed to send message to consumer.", queueName, consumerTag, e); + sendFuture.completeExceptionally(e); + return; + } finally { + msg.release(); + } - if (autoAck) { - messageAck(index.getPosition()); - } - } finally { - index.release(); - indexMessage.recycle(); - } - })).exceptionally(throwable -> { + if (autoAck) { + messageAck(index.getPosition()); + } + } finally { + index.release(); + indexMessage.recycle(); + } + })).exceptionally(throwable -> { log.error("[{}-{}] Failed to read data from exchange topic {}.", queueName, consumerTag, indexMessage.getExchangeName(), throwable); sendFuture.completeExceptionally(throwable); return null; - }); + }); return sendFuture; } @@ -259,7 +260,7 @@ public int getAvailablePermits() { return availablePermits; } return this.channel.getCreditManager().hasCredit() - ? (int) this.channel.getCreditManager().getMessageCredit() : 0; + ? (int) this.channel.getCreditManager().getMessageCredit() : 0; } @Override diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpExchange.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpExchange.java index 31c7eebc..ceefc749 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpExchange.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpExchange.java @@ -165,4 +165,7 @@ default CompletableFuture queueUnBind(String queue, String routingKey, Map new NotSupportedOperationException("Amqp exchange queue unbind operation is not supported.")); } + default void close(){ + + } } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpMultiBundlesChannel.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpMultiBundlesChannel.java index 6ae457de..1a14d610 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpMultiBundlesChannel.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpMultiBundlesChannel.java @@ -3,7 +3,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -14,7 +14,7 @@ package io.streamnative.pulsar.handlers.amqp; import static org.apache.qpid.server.protocol.ErrorCodes.INTERNAL_ERROR; - +import io.netty.util.ReferenceCountUtil; import io.streamnative.pulsar.handlers.amqp.admin.AmqpAdmin; import io.streamnative.pulsar.handlers.amqp.admin.model.BindingParams; import io.streamnative.pulsar.handlers.amqp.admin.model.ExchangeDeclareParams; @@ -25,20 +25,28 @@ import io.streamnative.pulsar.handlers.amqp.utils.MessageConvertUtils; import java.io.UnsupportedEncodingException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import lombok.extern.log4j.Log4j2; import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.MemoryLimitController; +import org.apache.pulsar.client.impl.MessageImpl; +import org.apache.pulsar.client.impl.ProducerImpl; +import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.util.FutureUtil; import org.apache.qpid.server.exchange.ExchangeDefaults; @@ -48,10 +56,13 @@ import org.apache.qpid.server.protocol.v0_8.IncomingMessage; import org.apache.qpid.server.protocol.v0_8.transport.AMQMethodBody; import org.apache.qpid.server.protocol.v0_8.transport.BasicAckBody; +import org.apache.qpid.server.protocol.v0_8.transport.BasicCancelOkBody; import org.apache.qpid.server.protocol.v0_8.transport.BasicConsumeOkBody; +import org.apache.qpid.server.protocol.v0_8.transport.ExchangeDeleteOkBody; import org.apache.qpid.server.protocol.v0_8.transport.MessagePublishInfo; import org.apache.qpid.server.protocol.v0_8.transport.MethodRegistry; import org.apache.qpid.server.protocol.v0_8.transport.QueueDeclareOkBody; +import org.apache.qpid.server.protocol.v0_8.transport.QueueDeleteOkBody; /** * Amqp Channel level method processor. @@ -59,13 +70,15 @@ @Log4j2 public class AmqpMultiBundlesChannel extends AmqpChannel { - private final Map> producerMap; private final List consumerList; + private volatile String defQueue; + private final PulsarClientImpl pulsarClient; + public AmqpMultiBundlesChannel(int channelId, AmqpConnection connection, AmqpBrokerService amqpBrokerService) { super(channelId, connection, amqpBrokerService); - this.producerMap = new ConcurrentHashMap<>(); this.consumerList = new ArrayList<>(); + this.pulsarClient = (PulsarClientImpl) this.connection.getAmqpBrokerService().getPulsarClient(); } @Override @@ -79,7 +92,7 @@ public void receiveExchangeDeclare(AMQShortString exchange, AMQShortString type, } ExchangeDeclareParams params = new ExchangeDeclareParams(); - params.setType(type != null ? type.toString() : null); + params.setType(type != null ? type.toString().toLowerCase() : null); params.setInternal(internal); params.setAutoDelete(autoDelete); params.setDurable(durable); @@ -87,7 +100,7 @@ public void receiveExchangeDeclare(AMQShortString exchange, AMQShortString type, params.setArguments(FieldTable.convertToMap(arguments)); getAmqpAdmin().exchangeDeclare( - connection.getNamespaceName().toString(), exchange.toString(), params).thenAccept(__ -> { + connection.getNamespaceName(), exchange.toString(), params).thenAccept(__ -> { if (!nowait) { connection.writeFrame( connection.getMethodRegistry().createExchangeDeclareOkBody().generateFrame(channelId)); @@ -114,15 +127,17 @@ public void receiveQueueDeclare(AMQShortString queue, boolean passive, boolean d params.setDurable(durable); params.setExclusive(exclusive); params.setAutoDelete(autoDelete); + params.setPassive(passive); params.setArguments(FieldTable.convertToMap(arguments)); - getAmqpAdmin().queueDeclare( - connection.getNamespaceName().toString(), queue.toString(), params).thenAccept(amqpQueue -> { -// setDefaultQueue(amqpQueue); - MethodRegistry methodRegistry = connection.getMethodRegistry(); - QueueDeclareOkBody responseBody = methodRegistry.createQueueDeclareOkBody( - AMQShortString.createAMQShortString(queue.toString()), 0, 0); - connection.writeFrame(responseBody.generateFrame(channelId)); + connection.getNamespaceName(), queue.toString(), params).thenAccept(amqpQueue -> { + setDefQueue(queue.toString()); + if (!nowait) { + MethodRegistry methodRegistry = connection.getMethodRegistry(); + QueueDeclareOkBody responseBody = methodRegistry.createQueueDeclareOkBody( + AMQShortString.createAMQShortString(queue.toString()), 0, 0); + connection.writeFrame(responseBody.generateFrame(channelId)); + } }).exceptionally(t -> { log.error("Failed to declare queue {} in vhost {}", queue, connection.getNamespaceName(), t); handleAoPException(t); @@ -139,19 +154,23 @@ public void receiveQueueBind(AMQShortString queue, AMQShortString exchange, AMQS } BindingParams params = new BindingParams(); - params.setRoutingKey(bindingKey != null ? bindingKey.toString() : null); + params.setRoutingKey(bindingKey != null ? bindingKey.toString() : ""); params.setArguments(FieldTable.convertToMap(argumentsTable)); - getAmqpAdmin().queueBind(connection.getNamespaceName().toString(), - exchange.toString(), queue.toString(), params).thenAccept(__ -> { - MethodRegistry methodRegistry = connection.getMethodRegistry(); - AMQMethodBody responseBody = methodRegistry.createQueueBindOkBody(); - connection.writeFrame(responseBody.generateFrame(channelId)); - }).exceptionally(t -> { - log.error("Failed to bind queue {} to exchange {}.", queue, exchange, t); - handleAoPException(t); - return null; - }); + AMQShortString finalQueue = getDefQueue(queue); + getAmqpAdmin().queueBind(connection.getNamespaceName(), + exchange.toString(), finalQueue.toString(), params) + .thenAccept(__ -> { + if (!nowait) { + MethodRegistry methodRegistry = connection.getMethodRegistry(); + AMQMethodBody responseBody = methodRegistry.createQueueBindOkBody(); + connection.writeFrame(responseBody.generateFrame(channelId)); + } + }).exceptionally(t -> { + log.error("Failed to bind queue {} to exchange {}.", finalQueue, exchange, t); + handleAoPException(t); + return null; + }); } @Override @@ -162,23 +181,76 @@ public void receiveQueueUnbind(AMQShortString queue, AMQShortString exchange, AM queue, exchange, bindingKey, arguments); } - getAmqpAdmin().queueUnbind(connection.getNamespaceName().toString(), exchange.toString(), - queue.toString(), bindingKey.toString()).thenAccept(__ -> { - AMQMethodBody responseBody = connection.getMethodRegistry().createQueueUnbindOkBody(); - connection.writeFrame(responseBody.generateFrame(channelId)); - }).exceptionally(t -> { - log.error("Failed to unbind queue {} with exchange {} in vhost {}", - queue, exchange, connection.getNamespaceName(), t); - handleAoPException(t); - return null; - }); + getAmqpAdmin().queueUnbind(connection.getNamespaceName(), exchange.toString(), + queue.toString(), bindingKey.toString()) + .thenAccept(__ -> { + AMQMethodBody responseBody = connection.getMethodRegistry().createQueueUnbindOkBody(); + connection.writeFrame(responseBody.generateFrame(channelId)); + }).exceptionally(t -> { + log.error("Failed to unbind queue {} with exchange {} in vhost {}", + queue, exchange, connection.getNamespaceName(), t); + handleAoPException(t); + return null; + }); + } + + @Override + public void receiveQueueDelete(AMQShortString queue, boolean ifUnused, boolean ifEmpty, boolean nowait) { + if (log.isDebugEnabled()) { + log.debug("RECV[{}] QueueDelete[ queue: {}, ifUnused:{}, ifEmpty:{}, nowait:{} ]", channelId, queue, + ifUnused, ifEmpty, nowait); + } + Map params = new HashMap<>(4); + params.put("if-unused", ifUnused); + params.put("if-empty", ifEmpty); + params.put("mode", "delete"); + params.put("name", queue.toString()); + params.put("vhost", connection.getNamespaceName().getLocalName()); + getAmqpAdmin().queueDelete(connection.getNamespaceName(), queue.toString(), params) + .thenAccept(__ -> { + if (!nowait) { + MethodRegistry methodRegistry = connection.getMethodRegistry(); + QueueDeleteOkBody responseBody = methodRegistry.createQueueDeleteOkBody(0); + connection.writeFrame(responseBody.generateFrame(channelId)); + } + }) + .exceptionally(t -> { + log.error("Failed to delete queue " + queue, t); + handleAoPException(t); + return null; + }); + } + + @Override + public void receiveExchangeDelete(AMQShortString exchange, boolean ifUnused, boolean nowait) { + if (log.isDebugEnabled()) { + log.debug("RECV[{}] receiveExchangeDelete[ exchange: {}, ifUnused:{} ]", channelId, + ifUnused, nowait); + } + Map params = new HashMap<>(2); + params.put("if-unused", ifUnused); + params.put("name", exchange.toString()); + params.put("vhost", connection.getNamespaceName().getLocalName()); + getAmqpAdmin().exchangeDelete(connection.getNamespaceName(), exchange.toString(), params) + .thenAccept(__ -> { + if (!nowait) { + ExchangeDeleteOkBody responseBody = connection.getMethodRegistry().createExchangeDeleteOkBody(); + connection.writeFrame(responseBody.generateFrame(channelId)); + } + }) + .exceptionally(t -> { + log.error("Failed to delete exchange {} in vhost {}.", + exchange, connection.getNamespaceName(), t); + handleAoPException(t); + return null; + }); } @Override public void receiveBasicQos(long prefetchSize, int prefetchCount, boolean global) { if (log.isDebugEnabled()) { log.debug("RECV[{}] BasicQos[prefetchSize: {} prefetchCount: {} global: {}]", - channelId, prefetchSize, prefetchCount, global); + channelId, prefetchSize, prefetchCount, global); } // ignored this method first @@ -205,6 +277,10 @@ public void receiveBasicConsume(AMQShortString queue, AMQShortString consumerTag connection.writeFrame(basicConsumeOkBody.generateFrame(channelId)); } consumer.startConsume(); + }).exceptionally(t -> { + log.error("Failed to create consumer {} in vhost {}.", queue, connection.getNamespaceName(), t); + handleAoPException(t); + return null; }); } @@ -215,40 +291,48 @@ public void receiveBasicPublish(AMQShortString exchange, AMQShortString routingK log.debug("RECV[{}] BasicPublish[exchange: {} routingKey: {} mandatory: {} immediate: {}]", channelId, exchange, routingKey, mandatory, immediate); } + AMQShortString routingKeyLocal = routingKey == null ? AMQShortString.valueOf("") : routingKey; if (isDefaultExchange(exchange)) { - ExchangeDeclareParams exchangeParams = new ExchangeDeclareParams(); - exchangeParams.setType(ExchangeDefaults.DIRECT_EXCHANGE_CLASS); - exchangeParams.setInternal(false); - exchangeParams.setAutoDelete(false); - exchangeParams.setDurable(true); - exchangeParams.setPassive(false); - getAmqpAdmin().exchangeDeclare(connection.getNamespaceName().toString(), - AbstractAmqpExchange.DEFAULT_EXCHANGE_DURABLE, exchangeParams - ).thenCompose(__ -> { - QueueDeclareParams queueParams = new QueueDeclareParams(); - queueParams.setDurable(true); - queueParams.setExclusive(false); - queueParams.setAutoDelete(false); - return getAmqpAdmin().queueDeclare(connection.getNamespaceName().toString(), routingKey.toString(), - queueParams); - }).thenCompose(__ -> { - BindingParams bindingParams = new BindingParams(); - bindingParams.setRoutingKey(routingKey.toString()); - return getAmqpAdmin().queueBind(connection.getNamespaceName().toString(), - AbstractAmqpExchange.DEFAULT_EXCHANGE_DURABLE, routingKey.toString(), bindingParams); - }).thenRun(() -> { - MessagePublishInfo info = - new MessagePublishInfo(AMQShortString.valueOf(AbstractAmqpExchange.DEFAULT_EXCHANGE_DURABLE), - immediate, mandatory, routingKey); - setPublishFrame(info, null); - }).exceptionally(t -> { - log.error("Failed to bind queue {} to exchange {}", routingKey, - AbstractAmqpExchange.DEFAULT_EXCHANGE_DURABLE, t); - handleAoPException(t); - return null; - }).join(); + MessagePublishInfo messagePublishInfo = connection.publishInfoMap.get(routingKeyLocal.toString()); + if (messagePublishInfo != null) { + setPublishFrame(messagePublishInfo, null); + return; + } + synchronized (connection) { + messagePublishInfo = connection.publishInfoMap.get(routingKeyLocal.toString()); + if (messagePublishInfo != null) { + setPublishFrame(messagePublishInfo, null); + return; + } + ExchangeDeclareParams exchangeParams = new ExchangeDeclareParams(); + exchangeParams.setType(ExchangeDefaults.DIRECT_EXCHANGE_CLASS); + exchangeParams.setInternal(false); + exchangeParams.setAutoDelete(false); + exchangeParams.setDurable(true); + exchangeParams.setPassive(false); + getAmqpAdmin().exchangeDeclare(connection.getNamespaceName(), + AbstractAmqpExchange.DEFAULT_EXCHANGE_DURABLE, exchangeParams + ).thenCompose(__ -> { + BindingParams bindingParams = new BindingParams(); + bindingParams.setRoutingKey(routingKeyLocal.toString()); + return getAmqpAdmin().queueBind(connection.getNamespaceName(), + AbstractAmqpExchange.DEFAULT_EXCHANGE_DURABLE, routingKeyLocal.toString(), bindingParams); + }).thenRun(() -> { + MessagePublishInfo info = + new MessagePublishInfo( + AMQShortString.valueOf(AbstractAmqpExchange.DEFAULT_EXCHANGE_DURABLE), + immediate, mandatory, routingKeyLocal); + connection.publishInfoMap.putIfAbsent(routingKeyLocal.toString(), info); + setPublishFrame(connection.publishInfoMap.get(routingKeyLocal.toString()), null); + }).exceptionally(t -> { + log.error("Failed to bind queue {} to exchange {}", routingKeyLocal, + AbstractAmqpExchange.DEFAULT_EXCHANGE_DURABLE, t); + handleAoPException(t); + return null; + }).join(); + } } else { - MessagePublishInfo info = new MessagePublishInfo(exchange, immediate, mandatory, routingKey); + MessagePublishInfo info = new MessagePublishInfo(exchange, immediate, mandatory, routingKeyLocal); setPublishFrame(info, null); } } @@ -263,7 +347,7 @@ protected void deliverCurrentMessageIfComplete() { if (currentMessage.allContentReceived()) { MessagePublishInfo info = currentMessage.getMessagePublishInfo(); String exchangeName = AMQShortString.toString(info.getExchange()); - Message message; + MessageImpl message; try { message = MessageConvertUtils.toPulsarMessage(currentMessage); } catch (UnsupportedEncodingException e) { @@ -275,32 +359,25 @@ protected void deliverCurrentMessageIfComplete() { exchangeName = AbstractAmqpExchange.DEFAULT_EXCHANGE_DURABLE; } - Producer producer; - try { - producer = getProducer(exchangeName); - } catch (PulsarServerException e) { - log.error("Failed to create producer for exchange {}.", exchangeName, e); - connection.sendConnectionClose(INTERNAL_ERROR, - "Failed to create producer for exchange " + exchangeName + ".", channelId); - return; - } - producer.newMessage() - .value(message.getData()) - .properties(message.getProperties()) - .sendAsync() - .thenAccept(position -> { - if (log.isDebugEnabled()) { - log.debug("Publish message success, position {}", position); - } - if (confirmOnPublish) { - confirmedMessageCounter++; - BasicAckBody body = connection.getMethodRegistry(). - createBasicAckBody(confirmedMessageCounter, false); - connection.writeFrame(body.generateFrame(channelId)); - } - }) - .exceptionally(throwable -> { + CompletableFuture> producerFuture = getProducer(exchangeName); + producerFuture.thenCompose(producer -> { + ProducerImpl producerImpl = (ProducerImpl) producer; + return producerImpl.sendAsync(message) + .thenAccept(position -> { + if (log.isDebugEnabled()) { + log.debug("Publish message success, position {}", position); + } + if (confirmOnPublish) { + confirmedMessageCounter++; + BasicAckBody body = connection.getMethodRegistry(). + createBasicAckBody(confirmedMessageCounter, false); + connection.writeFrame(body.generateFrame(channelId)); + } + }); + }).exceptionally(throwable -> { + ReferenceCountUtil.safeRelease(message.getDataBuffer()); log.error("Failed to write message to exchange", throwable); + handleAoPException(throwable); return null; }); } @@ -309,21 +386,147 @@ protected void deliverCurrentMessageIfComplete() { @Override public void receiveBasicReject(long deliveryTag, boolean requeue) { // TODO handle message reject, message requeue - log.error("Not supported operation receiveBasicReject."); + super.messageNAck(deliveryTag, false, requeue); } @Override public void receiveBasicNack(long deliveryTag, boolean multiple, boolean requeue) { // TODO handle message negative ack, message requeue - log.error("Not supported operation receiveBasicNack."); + super.messageNAck(deliveryTag, multiple, requeue); + } + + @Override + public void receiveAccessRequest(AMQShortString realm, boolean exclusive, boolean passive, boolean active, + boolean write, boolean read) { + super.receiveAccessRequest(realm, exclusive, passive, active, write, read); + } + + @Override + public void receiveBasicAck(long deliveryTag, boolean multiple) { + super.receiveBasicAck(deliveryTag, multiple); + } + + @Override + public void receiveConfirmSelect(boolean nowait) { + super.receiveConfirmSelect(nowait); + } + + @Override + public void receiveExchangeBound(AMQShortString exchange, AMQShortString routingKey, AMQShortString queueName) { + connection.sendConnectionClose(INTERNAL_ERROR, "Not supported for use [receiveExchangeBound]", channelId); + } + + @Override + public void receiveQueuePurge(AMQShortString queue, boolean nowait) { + Map params = new HashMap<>(8); + params.put("mode", "purge"); + params.put("name", queue.toString()); + params.put("vhost", connection.getNamespaceName().getLocalName()); + getAmqpAdmin().queuePurge(connection.getNamespaceName(), queue.toString(), params) + .thenAccept(__ -> { + if (!nowait) { + MethodRegistry methodRegistry = connection.getMethodRegistry(); + AMQMethodBody responseBody = methodRegistry.createQueuePurgeOkBody(0); + connection.writeFrame(responseBody.generateFrame(channelId)); + } + }) + .exceptionally(t -> { + log.error("Failed to purge queue {} ", queue, t); + handleAoPException(t); + return null; + }); + } + + @Override + public void receiveBasicCancel(AMQShortString consumerTag, boolean noWait) { + consumerList.removeIf(amqpPulsarConsumer -> { + if (amqpPulsarConsumer.getConsumerTag().equals(AMQShortString.toString(consumerTag))) { + try { + amqpPulsarConsumer.close(); + // Start expiration detection + getAmqpAdmin().startExpirationDetection(connection.getNamespaceName(), + amqpPulsarConsumer.getQueue()); + } catch (Exception e) { + handleAoPException(e); + log.error("[receiveBasicCancel] Failed to close consumer. queue:{}", + amqpPulsarConsumer.getQueue(), e); + } + return true; + } + return false; + }); + if (!noWait) { + MethodRegistry methodRegistry = connection.getMethodRegistry(); + BasicCancelOkBody cancelOkBody = methodRegistry.createBasicCancelOkBody(consumerTag); + connection.writeFrame(cancelOkBody.generateFrame(channelId)); + } + } + + @Override + public void receiveBasicGet(AMQShortString queue, boolean noAck) { + Optional pulsarConsumer = consumerList.stream() + .filter(amqpPulsarConsumer -> amqpPulsarConsumer.getConsumerTag().equals("get-" + queue.toString())) + .findFirst(); + pulsarConsumer.ifPresentOrElse(amqpPulsarConsumer -> amqpPulsarConsumer.consumeOne(noAck), + () -> getConsumer(queue.toString(), getConsumerTag(AMQShortString.valueOf("get-" + queue)), noAck) + .thenAccept(amqpPulsarConsumer -> amqpPulsarConsumer.consumeOne(noAck)) + .exceptionally(throwable -> { + handleAoPException(throwable); + return null; + })); + } + + @Override + public void receiveChannelFlow(boolean active) { + // rabbitmq peer nodes pull data + super.receiveChannelFlow(active); + } + + @Override + public void receiveChannelFlowOk(boolean active) { + super.receiveChannelFlowOk(active); + } + + @Override + public void receiveChannelClose(int replyCode, AMQShortString replyText, int classId, int methodId) { + super.receiveChannelClose(replyCode, replyText, classId, methodId); + } + + @Override + public void receiveChannelCloseOk() { + super.receiveChannelCloseOk(); + } + + @Override + public void receiveBasicRecover(boolean requeue, boolean sync) { + super.receiveBasicRecover(requeue, sync); + } + + @Override + public void receiveTxSelect() { + super.receiveTxSelect(); + } + + @Override + public void receiveTxCommit() { + super.receiveTxCommit(); + } + + @Override + public void receiveTxRollback() { + super.receiveTxRollback(); + } + + @Override + public void receivedComplete() { + super.receivedComplete(); } @Override public void close() { closeAllConsumers(); - closeAllProducers(); // TODO need to delete exclusive queues in this channel. - setDefaultQueue(null); + setDefQueue(null); } private void closeAllConsumers() { @@ -338,6 +541,8 @@ private void closeAllConsumers() { consumerList.forEach(consumer -> { try { consumer.close(); + // Start expiration detection + getAmqpAdmin().startExpirationDetection(connection.getNamespaceName(), consumer.getQueue()); } catch (Exception e) { log.error("Failed to close consumer.", e); } @@ -348,70 +553,72 @@ private void closeAllConsumers() { } } - private void closeAllProducers() { - for (Producer producer : producerMap.values()) { - try { - producer.close(); - } catch (PulsarClientException e) { - log.error("Failed to close producer.", e); - } - } - } - private AmqpAdmin getAmqpAdmin() { return this.connection.getAmqpBrokerService().getAmqpAdmin(); } - public Producer getProducer(String exchange) throws PulsarServerException { - PulsarClient client = connection.getPulsarService().getClient(); - return producerMap.computeIfAbsent(exchange, k -> { - try { - return client.newProducer() - .topic(getTopicName(PersistentExchange.TOPIC_PREFIX, exchange)) - .enableBatching(false) - .create(); - } catch (PulsarClientException e) { - throw new AoPServiceRuntimeException.ProducerCreationRuntimeException(e); - } - }); + public CompletableFuture> getProducer(String exchange) { + return connection.producerMap.computeIfAbsent(exchange, + k -> getAmqpAdmin().loadExchange(connection.getNamespaceName(), exchange) + .thenCompose(__ -> pulsarClient.newProducer() + .topic(getTopicName(PersistentExchange.TOPIC_PREFIX, exchange)) + .enableBatching(false) + .blockIfQueueFull(true) + .maxPendingMessages(20000) + .sendTimeout(0, TimeUnit.MILLISECONDS) + .createAsync())); } public CompletableFuture getConsumer(String queue, String consumerTag, boolean autoAck) { - PulsarClient client; - try { - client = connection.getPulsarService().getClient(); - } catch (PulsarServerException e) { - return FutureUtil.failedFuture(e); - } CompletableFuture consumerFuture = new CompletableFuture<>(); - client.newConsumer() + getAmqpAdmin().loadQueue(connection.getNamespaceName(), queue).thenCompose(__ -> pulsarClient.newConsumer() .topic(getTopicName(PersistentQueue.TOPIC_PREFIX, queue)) .subscriptionType(SubscriptionType.Shared) + .property("client_ip", connection.getClientIp()) .subscriptionName("AMQP_DEFAULT") .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) .consumerName(UUID.randomUUID().toString()) + .poolMessages(true) .receiverQueueSize(getConnection().getAmqpConfig().getAmqpPulsarConsumerQueueSize()) .negativeAckRedeliveryDelay(0, TimeUnit.MILLISECONDS) .subscribeAsync() - .thenAccept(consumer-> { - AmqpPulsarConsumer amqpPulsarConsumer = new AmqpPulsarConsumer(consumerTag, consumer, autoAck, + .thenAccept(consumer -> { + AmqpPulsarConsumer amqpPulsarConsumer = new AmqpPulsarConsumer(queue, consumerTag, consumer, autoAck, AmqpMultiBundlesChannel.this, - AmqpMultiBundlesChannel.this.connection.getPulsarService().getExecutor()); - consumerFuture.complete(amqpPulsarConsumer); - consumerList.add(amqpPulsarConsumer); - }) - .exceptionally(t -> { - consumerFuture.completeExceptionally(t); - return null; - }); + AmqpMultiBundlesChannel.this.connection.getPulsarService(), getAmqpAdmin()); + try { + amqpPulsarConsumer.initDLQ() + .thenRun(() -> { + getAmqpAdmin().getQueueBindings(connection.getNamespaceName(), queue) + .thenAccept(queueBinds -> queueBinds.forEach( + queueBind -> getAmqpAdmin().loadExchange(connection.getNamespaceName(), + queueBind.getSource()))); + consumerFuture.complete(amqpPulsarConsumer); + consumerList.add(amqpPulsarConsumer); + }); + } catch (PulsarServerException | PulsarAdminException e) { + throw new RuntimeException(e); + } + })).exceptionally(throwable -> { + consumerFuture.completeExceptionally(throwable); + return null; + }); return consumerFuture; } private String getTopicName(String topicPrefix, String name) { return TopicDomain.persistent + "://" - + connection.getAmqpConfig().getAmqpTenant() + "/" + + connection.getNamespaceName().getTenant() + "/" + connection.getNamespaceName().getLocalName() + "/" + topicPrefix + name; } + public void setDefQueue(String queue) { + defQueue = queue; + } + + public AMQShortString getDefQueue(AMQShortString queue) { + return queue == null || queue.length() == 0 ? + defQueue != null ? AMQShortString.valueOf(defQueue) : null : queue; + } } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpPullConsumer.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpPullConsumer.java index b760e5e4..2e72d137 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpPullConsumer.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpPullConsumer.java @@ -28,14 +28,14 @@ public class AmqpPullConsumer extends AmqpConsumer { public AmqpPullConsumer(QueueContainer queueContainer, Subscription subscription, - CommandSubscribe.SubType subType, String topicName, long consumerId, int priorityLevel, - String consumerName, boolean isDurable, ServerCnx cnx, String appId, - Map metadata, boolean readCompacted, MessageId messageId, - KeySharedMeta keySharedMeta, AmqpChannel channel, String consumerTag, String queueName, - boolean autoAck) { + CommandSubscribe.SubType subType, String topicName, long consumerId, int priorityLevel, + String consumerName, boolean isDurable, ServerCnx cnx, String appId, + Map metadata, boolean readCompacted, MessageId messageId, + KeySharedMeta keySharedMeta, AmqpChannel channel, String consumerTag, String queueName, + boolean autoAck) { super(queueContainer, subscription, subType, topicName, consumerId, priorityLevel, consumerName, isDurable, cnx, appId, metadata, readCompacted, messageId, keySharedMeta, channel, - consumerTag, queueName, autoAck); + consumerTag, queueName, autoAck); } @Override diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpPulsarConsumer.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpPulsarConsumer.java index 2543a4d2..7a26f29d 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpPulsarConsumer.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpPulsarConsumer.java @@ -3,7 +3,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,20 +13,44 @@ */ package io.streamnative.pulsar.handlers.amqp; +import io.netty.util.ReferenceCountUtil; +import io.streamnative.pulsar.handlers.amqp.admin.AmqpAdmin; +import io.streamnative.pulsar.handlers.amqp.impl.PersistentExchange; +import io.streamnative.pulsar.handlers.amqp.impl.PersistentQueue; import io.streamnative.pulsar.handlers.amqp.utils.MessageConvertUtils; +import io.streamnative.pulsar.handlers.amqp.utils.QueueUtil; +import io.streamnative.pulsar.handlers.amqp.utils.TopicUtil; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.client.impl.BackoffBuilder; import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.client.impl.MessageImpl; +import org.apache.pulsar.client.impl.ProducerImpl; +import org.apache.pulsar.client.impl.TypedMessageBuilderImpl; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicName; import org.apache.qpid.server.protocol.v0_8.AMQShortString; +import org.apache.qpid.server.protocol.v0_8.transport.BasicGetEmptyBody; +import org.apache.qpid.server.protocol.v0_8.transport.MethodRegistry; /** * AMQP Pulsar consumer. @@ -34,21 +58,36 @@ @Slf4j public class AmqpPulsarConsumer implements UnacknowledgedMessageMap.MessageProcessor { + @Getter private final String consumerTag; + @Getter private final Consumer consumer; private final AmqpChannel amqpChannel; private final ScheduledExecutorService executorService; private final boolean autoAck; private volatile boolean isClosed = false; private final Backoff consumeBackoff; + private CompletableFuture> producer; + private PulsarAdmin pulsarAdmin; + private String routingKey; + private String dleExchangeName; + @Getter + private final String queue; - public AmqpPulsarConsumer(String consumerTag, Consumer consumer, boolean autoAck, AmqpChannel amqpChannel, - ScheduledExecutorService executorService) { + private final PulsarService pulsarService; + private final AmqpAdmin amqpAdmin; + + public AmqpPulsarConsumer(String queue, String consumerTag, Consumer consumer, boolean autoAck, + AmqpChannel amqpChannel, + PulsarService pulsarService, AmqpAdmin amqpAdmin) { + this.queue = queue; this.consumerTag = consumerTag; this.consumer = consumer; this.autoAck = autoAck; this.amqpChannel = amqpChannel; - this.executorService = executorService; + this.pulsarService = pulsarService; + this.amqpAdmin = amqpAdmin; + this.executorService = pulsarService.getExecutor(); this.consumeBackoff = new BackoffBuilder() .setInitialTime(1, TimeUnit.MILLISECONDS) .setMax(1, TimeUnit.SECONDS) @@ -56,16 +95,95 @@ public AmqpPulsarConsumer(String consumerTag, Consumer consumer, boolean .create(); } + public CompletableFuture initDLQ() throws PulsarAdminException, PulsarServerException { + this.pulsarAdmin = pulsarService.getAdminClient(); + Map properties = pulsarAdmin.topics().getProperties(consumer.getTopic()); + String args = properties.get(PersistentQueue.ARGUMENTS); + if (StringUtils.isNotBlank(args)) { + Map arguments = QueueUtil.covertStringValueAsObjectMap(args); + Object dleExchangeName; + String dleName; + this.routingKey = (String) arguments.get("x-dead-letter-routing-key"); + if ((dleExchangeName = arguments.get(PersistentQueue.X_DEAD_LETTER_EXCHANGE)) != null + && StringUtils.isNotBlank(dleName = dleExchangeName.toString())) { + if (StringUtils.isBlank(routingKey)) { + this.routingKey = ""; + } + NamespaceName namespaceName = TopicName.get(consumer.getTopic()).getNamespaceObject(); + String topic = TopicUtil.getTopicName(PersistentExchange.TOPIC_PREFIX, + namespaceName.getTenant(), namespaceName.getLocalName(), dleName); + this.dleExchangeName = dleExchangeName.toString(); + return amqpAdmin.loadExchange(namespaceName, this.dleExchangeName) + .thenCompose(__ -> { + try { + this.producer = pulsarService.getClient().newProducer() + .topic(topic) + .enableBatching(false) + .createAsync(); + } catch (PulsarServerException e) { + throw new RuntimeException(e); + } + return producer; + }) + .thenApply(__ -> null); + } + } + return CompletableFuture.completedFuture(null); + } + public void startConsume() { executorService.submit(this::consume); } + public void consumeOne(boolean noAck) { + MessageImpl message; + try { + message = (MessageImpl) this.consumer.receive(1, TimeUnit.SECONDS); + } catch (PulsarClientException e) { + log.error("Failed to receive message and send to client", e); + amqpChannel.close(); + return; + } + if (message == null) { + MethodRegistry methodRegistry = amqpChannel.getConnection().getMethodRegistry(); + BasicGetEmptyBody responseBody = methodRegistry.createBasicGetEmptyBody(null); + amqpChannel.getConnection().writeFrame(responseBody.generateFrame(amqpChannel.getChannelId())); + return; + } + MessageIdImpl messageId = (MessageIdImpl) message.getMessageId(); + long deliveryIndex = this.amqpChannel.getNextDeliveryTag(); + try { + this.amqpChannel.getConnection().getAmqpOutputConverter().writeGetOk( + MessageConvertUtils.messageToAmqpBody(message), + this.amqpChannel.getChannelId(), + false, + deliveryIndex, 0); + } catch (Exception e) { + log.error("Unknown exception", e); + amqpChannel.close(); + return; + } finally { + message.release(); + } + if (noAck) { + this.consumer.acknowledgeAsync(messageId).exceptionally(t -> { + log.error("Failed to ack message {} for topic {} by auto ack.", + messageId, consumer.getTopic(), t); + return null; + }); + } else { + this.amqpChannel.getUnacknowledgedMessageMap().add( + deliveryIndex, PositionImpl.get(messageId.getLedgerId(), messageId.getEntryId()), + AmqpPulsarConsumer.this, 0); + } + } + private void consume() { if (isClosed) { return; } - Message message; + Message message = null; try { message = this.consumer.receive(0, TimeUnit.SECONDS); if (message == null) { @@ -75,12 +193,16 @@ private void consume() { MessageIdImpl messageId = (MessageIdImpl) message.getMessageId(); long deliveryIndex = this.amqpChannel.getNextDeliveryTag(); - this.amqpChannel.getConnection().getAmqpOutputConverter().writeDeliver( - MessageConvertUtils.messageToAmqpBody(message), - this.amqpChannel.getChannelId(), - false, - deliveryIndex, - AMQShortString.createAMQShortString(this.consumerTag)); + try { + this.amqpChannel.getConnection().getAmqpOutputConverter().writeDeliver( + MessageConvertUtils.messageToAmqpBody(message), + this.amqpChannel.getChannelId(), + false, + deliveryIndex, + AMQShortString.createAMQShortString(this.consumerTag)); + } finally { + message.release(); + } if (this.autoAck) { this.consumer.acknowledgeAsync(messageId).exceptionally(t -> { log.error("Failed to ack message {} for topic {} by auto ack.", @@ -90,11 +212,14 @@ private void consume() { } else { this.amqpChannel.getUnacknowledgedMessageMap().add( deliveryIndex, PositionImpl.get(messageId.getLedgerId(), messageId.getEntryId()), - AmqpPulsarConsumer.this, message.size()); + AmqpPulsarConsumer.this, 0); } consumeBackoff.reset(); - this.consume(); + this.executorService.execute(this::consume); } catch (Exception e) { + if (message != null) { + ReferenceCountUtil.safeRelease(((MessageImpl) message).getDataBuffer()); + } long backoff = consumeBackoff.next(); log.error("Failed to receive message and send to client, retry in {} ms.", backoff, e); this.executorService.schedule(this::consume, backoff, TimeUnit.MILLISECONDS); @@ -113,10 +238,46 @@ public void requeue(List positions) { } } + @Override + public void discardMessage(List positions) { + if (producer == null) { + for (PositionImpl pos : positions) { + consumer.acknowledgeAsync(new MessageIdImpl(pos.getLedgerId(), pos.getEntryId(), -1)); + } + return; + } + for (PositionImpl pos : positions) { + pulsarAdmin.topics().getMessageByIdAsync(consumer.getTopic(), pos.getLedgerId(), pos.getEntryId()) + .thenAccept(message -> { + Map properties = new HashMap<>(message.getProperties()); + properties.put(MessageConvertUtils.PROP_ROUTING_KEY, routingKey); + properties.put(MessageConvertUtils.PROP_EXCHANGE, dleExchangeName); + properties.put(MessageConvertUtils.PROP_EXPIRATION, "0"); + TypedMessageBuilderImpl messageBuilder = + new TypedMessageBuilderImpl<>(null, Schema.BYTES); + messageBuilder.properties(properties); + messageBuilder.value(message.getValue()); + producer.thenAccept(p -> { + if (p instanceof ProducerImpl producerImpl) { + producerImpl.sendAsync(messageBuilder.getMessage()) + .thenCompose(messageId -> consumer.acknowledgeAsync( + new MessageIdImpl(pos.getLedgerId(), pos.getEntryId(), -1))); + } + }); + }).exceptionally(throwable -> { + log.error("Query message [{}] fail", pos, throwable); + return null; + }); + } + } + public void close() throws PulsarClientException { this.isClosed = true; this.consumer.pause(); this.consumer.close(); + if (producer != null) { + producer.thenApply(Producer::closeAsync); + } } } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpQueue.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpQueue.java index 9dbf1dd2..7335ac03 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpQueue.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpQueue.java @@ -23,7 +23,7 @@ * Interface of the AMQP queue. * The AMQP broker should maintaining queues in a Map, so that the broker can find the right queue to read messages. */ -public interface AmqpQueue { +public interface AmqpQueue extends AutoCloseable{ /** * Get name of the queue. @@ -33,7 +33,9 @@ public interface AmqpQueue { String getName(); boolean getDurable(); - + boolean getExclusive(); + boolean getAutoDelete(); + Map getArguments(); /** * Write the index message into the queue. */ @@ -67,7 +69,7 @@ CompletableFuture writeIndexMessageAsync(String exchangeName, long ledgerI * Bind to a exchange {@link AmqpExchange}. */ CompletableFuture bindExchange(AmqpExchange exchange, AmqpMessageRouter router, String bindingKey, - Map arguments); + Map arguments); /** * UnBind a exchange for the queue. @@ -94,4 +96,7 @@ CompletableFuture bindExchange(AmqpExchange exchange, AmqpMessageRouter ro boolean isAutoDelete(); Topic getTopic(); + + @Override + void close(); } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpServiceConfiguration.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpServiceConfiguration.java index 8e6c4afd..aba4b15b 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpServiceConfiguration.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/AmqpServiceConfiguration.java @@ -137,4 +137,9 @@ public class AmqpServiceConfiguration extends ServiceConfiguration { ) private int amqpPulsarConsumerQueueSize = 10000; + @FieldContext( + category = CATEGORY_AMQP, + required = true + ) + private String amqpServerAddress; } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/ConnectionContainer.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/ConnectionContainer.java index a85fcb8c..612272b0 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/ConnectionContainer.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/ConnectionContainer.java @@ -31,9 +31,12 @@ public class ConnectionContainer { private Map> connectionMap = Maps.newConcurrentMap(); + private final AmqpServiceConfiguration config; protected ConnectionContainer(PulsarService pulsarService, - ExchangeContainer exchangeContainer, QueueContainer queueContainer) { + ExchangeContainer exchangeContainer, + QueueContainer queueContainer, AmqpServiceConfiguration config) { + this.config = config; pulsarService.getNamespaceService().addNamespaceBundleOwnershipListener(new NamespaceBundleOwnershipListener() { @Override public void onLoad(NamespaceBundle namespaceBundle) { @@ -48,17 +51,19 @@ public void unLoad(NamespaceBundle namespaceBundle) { log.info("ConnectionContainer [unLoad] namespaceBundle: {}", namespaceBundle); NamespaceName namespaceName = namespaceBundle.getNamespaceObject(); - if (connectionMap.containsKey(namespaceName)) { - Set connectionSet = connectionMap.get(namespaceName); - for (AmqpConnection connection : connectionSet) { - log.info("close connection: {}", connection); - if (connection.getOrderlyClose().compareAndSet(false, true)) { - connection.completeAndCloseAllChannels(); - connection.close(); + if (!config.isAmqpMultiBundleEnable()) { + if (connectionMap.containsKey(namespaceName)) { + Set connectionSet = connectionMap.get(namespaceName); + for (AmqpConnection connection : connectionSet) { + log.info("close connection: {}", connection); + if (connection.getOrderlyClose().compareAndSet(false, true)) { + connection.completeAndCloseAllChannels(); + connection.close(); + } } + connectionSet.clear(); + connectionMap.remove(namespaceName); } - connectionSet.clear(); - connectionMap.remove(namespaceName); } if (exchangeContainer.getExchangeMap().containsKey(namespaceName)) { diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/ExchangeContainer.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/ExchangeContainer.java index 194185ea..73e74079 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/ExchangeContainer.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/ExchangeContainer.java @@ -3,7 +3,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -22,7 +22,9 @@ import static io.streamnative.pulsar.handlers.amqp.utils.ExchangeUtil.getExchangeType; import static io.streamnative.pulsar.handlers.amqp.utils.ExchangeUtil.isBuildInExchange; +import io.streamnative.pulsar.handlers.amqp.admin.AmqpAdmin; import io.streamnative.pulsar.handlers.amqp.common.exception.AoPException; +import io.streamnative.pulsar.handlers.amqp.common.exception.AoPServiceRuntimeException; import io.streamnative.pulsar.handlers.amqp.impl.PersistentExchange; import io.streamnative.pulsar.handlers.amqp.utils.ExchangeUtil; import java.util.Collections; @@ -38,6 +40,7 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.qpid.server.protocol.ErrorCodes; @@ -51,13 +54,18 @@ public class ExchangeContainer { private PulsarService pulsarService; private final ExecutorService routeExecutor; private final AmqpServiceConfiguration config; + private final AmqpAdmin amqpAdmin; + private final PulsarClient pulsarClient; protected ExchangeContainer(AmqpTopicManager amqpTopicManager, PulsarService pulsarService, - ExecutorService routeExecutor, AmqpServiceConfiguration config) { + ExecutorService routeExecutor, AmqpServiceConfiguration config, AmqpAdmin amqpAdmin, + PulsarClient pulsarClient) { this.amqpTopicManager = amqpTopicManager; this.pulsarService = pulsarService; this.routeExecutor = routeExecutor; this.config = config; + this.amqpAdmin = amqpAdmin; + this.pulsarClient = pulsarClient; } @Getter @@ -68,18 +76,22 @@ public CompletableFuture asyncGetExchange(NamespaceName namespaceN String exchangeName, boolean createIfMissing, String exchangeType) { + if (isBuildInExchange(exchangeName)) { + createIfMissing = true; + exchangeType = getExchangeType(exchangeName); + } return asyncGetExchange(namespaceName, exchangeName, createIfMissing, exchangeType, true, false, false, null); } /** * Get or create exchange. * - * @param namespaceName namespace used in pulsar - * @param exchangeName name of exchange + * @param namespaceName namespace used in pulsar + * @param exchangeName name of exchange * @param createIfMissing true to create the exchange if not existed, and exchangeType should be not null * false to get the exchange and return null if not existed - * @param exchangeType type of exchange: direct,fanout,topic and headers - * @param arguments other properties (construction arguments) for the exchange + * @param exchangeType type of exchange: direct,fanout,topic and headers + * @param arguments other properties (construction arguments) for the exchange * @return the completableFuture of get result */ public CompletableFuture asyncGetExchange(NamespaceName namespaceName, @@ -90,17 +102,8 @@ public CompletableFuture asyncGetExchange(NamespaceName namespaceN boolean autoDelete, boolean internal, Map arguments) { - final boolean finalCreateIfMissing; - final String finalExchangeType; - if (isBuildInExchange(exchangeName)) { - finalCreateIfMissing = true; - finalExchangeType = getExchangeType(exchangeName); - } else { - finalCreateIfMissing = createIfMissing; - finalExchangeType = exchangeType; - } CompletableFuture amqpExchangeCompletableFuture = new CompletableFuture<>(); - if (StringUtils.isEmpty(finalExchangeType) && finalCreateIfMissing) { + if (StringUtils.isEmpty(exchangeType) && createIfMissing) { log.error("[{}][{}] ExchangeType should be set when createIfMissing is true.", namespaceName, exchangeName); amqpExchangeCompletableFuture.completeExceptionally( new IllegalArgumentException("exchangeType should be set when createIfMissing is true")); @@ -121,14 +124,26 @@ public CompletableFuture asyncGetExchange(NamespaceName namespaceN CompletableFuture existingAmqpExchangeFuture = exchangeMap.get(namespaceName). putIfAbsent(exchangeName, amqpExchangeCompletableFuture); if (existingAmqpExchangeFuture != null) { + if (createIfMissing) { + return existingAmqpExchangeFuture.thenCompose(amqpExchange -> { + if (amqpExchange instanceof PersistentExchange persistentExchange) { + if (!exchangeDeclareCheck( + amqpExchangeCompletableFuture, namespaceName.getLocalName(), + exchangeName, exchangeType, durable, autoDelete, persistentExchange.getProperties())) { + return amqpExchangeCompletableFuture; + } + } + return existingAmqpExchangeFuture; + }); + } return existingAmqpExchangeFuture; } else { String topicName = PersistentExchange.getExchangeTopicName(namespaceName, exchangeName); Map initProperties = new HashMap<>(); - if (finalCreateIfMissing) { + if (createIfMissing) { // if first create the exchange, try to set properties for exchange try { - initProperties = ExchangeUtil.generateTopicProperties(exchangeName, finalExchangeType, durable, + initProperties = ExchangeUtil.generateTopicProperties(exchangeName, exchangeType, durable, autoDelete, internal, arguments, Collections.EMPTY_LIST); } catch (Exception e) { log.error("Failed to generate topic properties for exchange {} in vhost {}.", @@ -140,7 +155,7 @@ public CompletableFuture asyncGetExchange(NamespaceName namespaceN } CompletableFuture topicCompletableFuture = amqpTopicManager.getTopic( - topicName, finalCreateIfMissing, initProperties); + topicName, createIfMissing, initProperties); topicCompletableFuture.whenComplete((topic, throwable) -> { if (throwable != null) { log.error("[{}][{}] Failed to get exchange topic.", namespaceName, exchangeName, throwable); @@ -149,15 +164,18 @@ public CompletableFuture asyncGetExchange(NamespaceName namespaceN } else { if (null == topic) { log.warn("[{}][{}] The exchange topic did not exist.", namespaceName, exchangeName); - amqpExchangeCompletableFuture.complete(null); + amqpExchangeCompletableFuture.completeExceptionally(new AoPServiceRuntimeException.NoSuchExchangeException( + "Exchange [" + exchangeName + "] not created")); removeExchangeFuture(namespaceName, exchangeName); + return; } else { // recover metadata if existed PersistentTopic persistentTopic = (PersistentTopic) topic; Map properties = persistentTopic.getManagedLedger().getProperties(); - if (finalCreateIfMissing && !exchangeDeclareCheck( + if (createIfMissing && !exchangeDeclareCheck( amqpExchangeCompletableFuture, namespaceName.getLocalName(), - exchangeName, finalExchangeType, durable, autoDelete, properties)) { + exchangeName, exchangeType, durable, autoDelete, properties)) { + removeExchangeFuture(namespaceName, exchangeName); return; } @@ -172,15 +190,18 @@ public CompletableFuture asyncGetExchange(NamespaceName namespaceN properties.getOrDefault(AUTO_DELETE, "false")); boolean currentInternal = Boolean.parseBoolean( properties.getOrDefault(INTERNAL, "false")); - amqpExchange = new PersistentExchange(exchangeName, + amqpExchange = new PersistentExchange(exchangeName, properties, AmqpExchange.Type.value(currentType), persistentTopic, currentDurable, currentAutoDelete, currentInternal, currentArguments, routeExecutor, - config.getAmqpExchangeRouteQueueSize(), config.isAmqpMultiBundleEnable()); + config.getAmqpExchangeRouteQueueSize(), config.isAmqpMultiBundleEnable(), + amqpAdmin, pulsarClient); } catch (Exception e) { log.error("Failed to init exchange {} in vhost {}.", exchangeName, namespaceName.getLocalName(), e); - amqpExchangeCompletableFuture.completeExceptionally(e); - removeExchangeFuture(namespaceName, exchangeName); + topic.deleteForcefully().thenRun(() -> { + removeExchangeFuture(namespaceName, exchangeName); + amqpExchangeCompletableFuture.completeExceptionally(e); + }); return; } amqpExchangeCompletableFuture.complete(amqpExchange); @@ -203,7 +224,7 @@ private boolean exchangeDeclareCheck(CompletableFuture exchangeFut String currentType = properties.get(TYPE); if (!StringUtils.equals(properties.get(TYPE), exchangeType)) { exchangeFuture.completeExceptionally(new AoPException(ErrorCodes.IN_USE, - String.format(replyTextFormat, "type", exchangeType, currentType), true, false)); + String.format(replyTextFormat, "type", exchangeType, currentType), true, true)); return false; } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/ExchangeMessageRouter.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/ExchangeMessageRouter.java index 6ff9c4fd..f22c0740 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/ExchangeMessageRouter.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/ExchangeMessageRouter.java @@ -3,7 +3,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,8 +16,8 @@ import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.FALSE; import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.TRUE; import static org.apache.pulsar.broker.service.persistent.PersistentTopic.MESSAGE_RATE_BACKOFF_MS; - import com.google.common.collect.Sets; +import io.netty.buffer.ByteBuf; import io.netty.util.ReferenceCountUtil; import io.streamnative.pulsar.handlers.amqp.common.exception.AoPServiceRuntimeException; import io.streamnative.pulsar.handlers.amqp.impl.HeadersMessageRouter; @@ -25,6 +25,7 @@ import io.streamnative.pulsar.handlers.amqp.impl.PersistentQueue; import io.streamnative.pulsar.handlers.amqp.utils.MessageConvertUtils; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -46,7 +47,8 @@ import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.pulsar.broker.PulsarServerException; +import org.apache.commons.lang.math.NumberUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.PulsarClient; @@ -74,7 +76,7 @@ public abstract class ExchangeMessageRouter { private final Map> producerMap = new ConcurrentHashMap<>(); private static final int defaultReadMaxSizeBytes = 5 * 1024 * 1024; - private static final int replicatorQueueSize = 1000; + private static final int replicatorQueueSize = 2000; private volatile int pendingQueueSize = 0; private static final AtomicIntegerFieldUpdater PENDING_SIZE_UPDATER = @@ -83,7 +85,9 @@ public abstract class ExchangeMessageRouter { private volatile int havePendingRead = FALSE; private static final AtomicIntegerFieldUpdater HAVE_PENDING_READ_UPDATER = AtomicIntegerFieldUpdater.newUpdater(ExchangeMessageRouter.class, "havePendingRead"); - + private volatile int isActive = FALSE; + private static final AtomicIntegerFieldUpdater ACTIVE_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(ExchangeMessageRouter.class, "isActive"); @AllArgsConstructor @EqualsAndHashCode private static class Destination { @@ -113,7 +117,9 @@ private void start0(ManagedLedgerImpl managedLedger) { public void openCursorComplete(ManagedCursor cursor, Object ctx) { log.info("Start to route messages for exchange {}", exchange.getName()); ExchangeMessageRouter.this.cursor = (ManagedCursorImpl) cursor; - readMoreEntries(); + if (ACTIVE_UPDATER.compareAndSet(ExchangeMessageRouter.this, FALSE, TRUE)) { + readMoreEntries(); + } } @Override @@ -121,59 +127,60 @@ public void openCursorFailed(ManagedLedgerException exception, Object ctx) { log.error("Failed to open cursor for exchange topic {}, retry", exchange.getName(), exception); start0(managedLedger); } - }, null); + }, null); } private void readMoreEntries() { - int availablePermits = getAvailablePermits(); - if (availablePermits > 0) { - if (HAVE_PENDING_READ_UPDATER.compareAndSet(this, FALSE, TRUE)) { - if (log.isDebugEnabled()) { - log.debug("{} Schedule read of {} messages.", exchange.getName(), availablePermits); - } - cursor.asyncReadEntriesOrWait(availablePermits, defaultReadMaxSizeBytes, - new AsyncCallbacks.ReadEntriesCallback() { - @Override - public void readEntriesComplete(List entries, Object ctx) { - HAVE_PENDING_READ_UPDATER.set(ExchangeMessageRouter.this, FALSE); - if (entries.size() == 0) { - log.warn("read empty entries, scheduled to read again."); - exchange.getTopic().getBrokerService().getPulsar().getExecutor() - .schedule(ExchangeMessageRouter.this::readMoreEntries, 1, TimeUnit.MILLISECONDS); - return; - } - routeExecutor.submit(() -> { - try { - routeMessages(entries); - } catch (Exception e) { - log.error("Failed to route messages.", e); - cursor.rewind(); - for (Entry entry : entries) { - ReferenceCountUtil.safeRelease(entry); - } - tryToReadMoreEntries(); + if (isActive == FALSE) { + return; + } + if (HAVE_PENDING_READ_UPDATER.compareAndSet(this, FALSE, TRUE)) { + cursor.asyncReadEntriesOrWait(replicatorQueueSize, defaultReadMaxSizeBytes, + new AsyncCallbacks.ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + if (entries.size() == 0) { + HAVE_PENDING_READ_UPDATER.set(ExchangeMessageRouter.this, FALSE); + log.warn("read empty entries, scheduled to read again."); + exchange.getTopic().getBrokerService().getPulsar().getExecutor() + .schedule(ExchangeMessageRouter.this::readMoreEntries, 1, + TimeUnit.MILLISECONDS); + return; } - }); - } + routeExecutor.submit(() -> { + try { + routeMessages(entries); + entries.clear(); + } catch (Exception e) { + log.error("Failed to route messages.", e); + if (e instanceof AoPServiceRuntimeException.ProducerCreationRuntimeException) { + producerMap.values().forEach(ProducerImpl::closeAsync); + producerMap.clear(); + } + cursor.rewind(); + for (Entry entry : entries) { + ReferenceCountUtil.safeRelease(entry); + } + } + HAVE_PENDING_READ_UPDATER.set(ExchangeMessageRouter.this, FALSE); + ExchangeMessageRouter.this.readMoreEntries(); + }); + } - @Override - public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { - HAVE_PENDING_READ_UPDATER.set(ExchangeMessageRouter.this, FALSE); - log.error("Failed to read entries from exchange {}", exchange.getName(), exception); - } - }, null, null); - } else { - log.warn("{} Not schedule read due to pending read. Messages to read {}.", - exchange.getName(), availablePermits); - if (log.isDebugEnabled()) { - log.debug("{} Not schedule read due to pending read. Messages to read {}.", - exchange.getName(), availablePermits); - } - } + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + HAVE_PENDING_READ_UPDATER.set(ExchangeMessageRouter.this, FALSE); + log.error("Failed to read entries from exchange {}", exchange.getName(), exception); + exchange.getTopic().getBrokerService().getPulsar().getExecutor() + .schedule(() -> { + cursor.rewind(); + readMoreEntries(); + }, MESSAGE_RATE_BACKOFF_MS, TimeUnit.MILLISECONDS); + } + }, null, null); } else { - // no permits from rate limit - exchange.getTopic().getBrokerService().getPulsar().getExecutor() - .schedule(this::readMoreEntries, MESSAGE_RATE_BACKOFF_MS, TimeUnit.MILLISECONDS); + log.warn("{} Not schedule read due to pending read.", + exchange.getName()); } } @@ -191,58 +198,73 @@ private int getAvailablePermits() { return availablePermits; } - private void routeMessages(List entries) throws PulsarServerException { - PENDING_SIZE_UPDATER.addAndGet(this, entries.size()); - for (Entry entry : entries) { - Map props; - MessageImpl message; - try { - message = MessageImpl.create(null, null, - Commands.parseMessageMetadata(entry.getDataBuffer()), - entry.getDataBuffer(), - Optional.empty(), null, Schema.BYTES, 0, true, -1L); - message.getMessageBuilder().clearSequenceId(); - props = message.getMessageBuilder().getPropertiesList().stream() - .collect(Collectors.toMap(KeyValue::getKey, KeyValue::getValue)); - } catch (Exception e) { - log.error("Deserialize entry dataBuffer failed for exchange {}, skip it first.", - exchange.getName(), e); - PENDING_SIZE_UPDATER.decrementAndGet(this); - entry.release(); - continue; - } + private void routeMessages(List entries) { + List positions = new ArrayList<>(entries.size()); + try { + for (Entry entry : entries) { + final Position position = entry.getPosition(); + ByteBuf dataBuffer = entry.getDataBuffer(); + Map props; + MessageImpl message; + try { + message = MessageImpl.create(null, null, Commands.parseMessageMetadata(dataBuffer), dataBuffer, + Optional.empty(), null, Schema.BYTES, 0, true, -1L); + props = message.getMessageBuilder().getPropertiesList().stream() + .collect(Collectors.toMap(KeyValue::getKey, KeyValue::getValue)); + } catch (Exception e) { + log.error("Deserialize entry dataBuffer failed for exchange {}, skip it first.", exchange.getName(), e); + entry.release(); + dataBuffer.release(); + continue; + } - Set destinations = getDestinations( - props.getOrDefault(MessageConvertUtils.PROP_ROUTING_KEY, ""), getMessageHeaders()); - initProducerIfNeeded(destinations); - - final Position position = entry.getPosition(); - List> futures = new ArrayList<>(); - if (!destinations.isEmpty()) { - final int readerIndex = message.getDataBuffer().readerIndex(); - for (Destination des : destinations) { - ProducerImpl producer = producerMap.get(des.name); - if (producer == null) { - log.error("Failed to get producer for des {}.", des.name); - throw new AoPServiceRuntimeException.MessageRouteException( - "Failed to get producer for des " + des.name + "."); + Set destinations = getDestinations( + props.getOrDefault(MessageConvertUtils.PROP_ROUTING_KEY, ""), getMessageHeaders()); + boolean notNull = destinations != null; + List> futures = notNull ? + new ArrayList<>(destinations.size()) : Collections.emptyList(); + try { + if (notNull && !destinations.isEmpty()) { + initProducerIfNeeded(destinations); + String xDelay; + int delay; + if (exchange.isExistDelayedType() + && StringUtils.isNotBlank(xDelay = props.get(MessageConvertUtils.BASIC_PROP_HEADER_X_DELAY)) + && NumberUtils.isNumber(xDelay) + && (delay = Integer.parseInt(xDelay)) > 0) { + message.getMessageBuilder().setDeliverAtTime(System.currentTimeMillis() + delay); + } + if (destinations.size() > 1) { + dataBuffer.retain(destinations.size() - 1); + } + final int readerIndex = dataBuffer.readerIndex(); + for (Destination des : destinations) { + ProducerImpl producer = producerMap.get(des.name); + message.getMessageBuilder().clearSequenceId(); + message.getMessageBuilder().clearProducerName(); + message.getMessageBuilder().clearPublishTime(); + dataBuffer.readerIndex(readerIndex); + futures.add(producer.sendAsync(message)); + } + } else { + dataBuffer.release(); } - message.getMessageBuilder().clearProducerName(); - message.getMessageBuilder().clearPublishTime(); - message.getDataBuffer().readerIndex(readerIndex); - futures.add(producer.sendAsync(message)); + } finally { + entry.release(); } + // If the producer creates an exception, add is not executed + positions.add(position); + FutureUtil.waitForAll(futures).exceptionally((t) -> { + if (t != null) { + log.error("Failed to route message {} for exchange {}.", position, exchange.exchangeName, t); + // TODO Application alarm notification needs to be added + } + return null; + }); } - entry.release(); - - FutureUtil.waitForAll(futures).whenComplete((__, t) -> { - if (t != null) { - log.error("Failed to route message {} for exchange {}.", position, exchange.exchangeName, t); - cursor.rewind(); - tryToReadMoreEntries(); - return; - } - cursor.asyncDelete(position, new AsyncCallbacks.DeleteCallback() { + } finally { + if (positions.size() != 0) { + cursor.asyncDelete(positions, new AsyncCallbacks.DeleteCallback() { @Override public void deleteComplete(Object ctx) { if (log.isDebugEnabled()) { @@ -254,9 +276,9 @@ public void deleteComplete(Object ctx) { public void deleteFailed(ManagedLedgerException exception, Object ctx) { log.error("{} Failed to delete message at {}", exchange.getName(), ctx, exception); } - }, position); - tryToReadMoreEntries(); - }); + }, null); + positions.clear(); + } } } @@ -267,8 +289,8 @@ private void tryToReadMoreEntries() { } } - private void initProducerIfNeeded(Set destinations) throws PulsarServerException { - PulsarClient pulsarClient = exchange.getTopic().getBrokerService().pulsar().getClient(); + private void initProducerIfNeeded(Set destinations) { + PulsarClient pulsarClient = exchange.getPulsarClient(); for (Destination des : destinations) { producerMap.computeIfAbsent(des.name, k -> { String topic = getTopic(des.name, des.type); @@ -276,6 +298,9 @@ private void initProducerIfNeeded(Set destinations) throws PulsarSe return (ProducerImpl) pulsarClient.newProducer() .topic(topic) .enableBatching(false) + .blockIfQueueFull(true) + .maxPendingMessages(20000) + .sendTimeout(0, TimeUnit.MILLISECONDS) .create(); } catch (PulsarClientException e) { throw new AoPServiceRuntimeException.ProducerCreationRuntimeException(e); @@ -300,6 +325,8 @@ public static ExchangeMessageRouter getInstance(PersistentExchange exchange, Exe case Direct -> new DirectExchangeMessageRouter(exchange, routeExecutor); case Topic -> new TopicExchangeMessageRouter(exchange, routeExecutor); case Headers -> new HeadersExchangeMessageRouter(exchange, routeExecutor); + default -> throw new AoPServiceRuntimeException.NotSupportedExchangeTypeException( + exchange.getType() + " not support"); }; } @@ -466,4 +493,9 @@ Set getDestinations(String routingKey, Map headers) } + public void close() { + ACTIVE_UPDATER.set(this, FALSE); + producerMap.values().forEach(ProducerImpl::closeAsync); + producerMap.clear(); + } } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/QueueContainer.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/QueueContainer.java index 10684da2..1bfcad68 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/QueueContainer.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/QueueContainer.java @@ -14,7 +14,11 @@ package io.streamnative.pulsar.handlers.amqp; +import io.streamnative.pulsar.handlers.amqp.common.exception.AoPException; +import io.streamnative.pulsar.handlers.amqp.common.exception.AoPServiceRuntimeException; import io.streamnative.pulsar.handlers.amqp.impl.PersistentQueue; +import io.streamnative.pulsar.handlers.amqp.utils.QueueUtil; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; @@ -26,6 +30,7 @@ import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.qpid.server.protocol.ErrorCodes; /** * Container for all queues in the broker. @@ -33,6 +38,7 @@ @Slf4j public class QueueContainer { + @Getter private AmqpTopicManager amqpTopicManager; private PulsarService pulsarService; private ExchangeContainer exchangeContainer; @@ -52,8 +58,8 @@ protected QueueContainer(AmqpTopicManager amqpTopicManager, PulsarService pulsar /** * Get or create queue. * - * @param namespaceName namespace in pulsar - * @param queueName name of queue + * @param namespaceName namespace in pulsar + * @param queueName name of queue * @param createIfMissing true to create the queue if not existed * false to get the queue and return null if not existed * @return the completableFuture of get result @@ -89,7 +95,9 @@ public CompletableFuture asyncGetQueue(NamespaceName namespaceName, S } else { if (null == topic) { log.warn("[{}][{}] Queue topic did not exist.", namespaceName, queueName); - queueCompletableFuture.complete(null); + queueCompletableFuture.completeExceptionally( + new AoPServiceRuntimeException.NoSuchQueueException( + "Queue [" + queueName + "] not created")); removeQueueFuture(namespaceName, queueName); } else { // recover metadata if existed @@ -98,7 +106,7 @@ public CompletableFuture asyncGetQueue(NamespaceName namespaceName, S // TODO: reset connectionId, exclusive and autoDelete PersistentQueue amqpQueue = new PersistentQueue(queueName, persistentTopic, - 0, false, false); + 0, false, false, properties); if (!config.isAmqpMultiBundleEnable()) { try { amqpQueue.recoverRoutersFromQueueProperties(properties, exchangeContainer, @@ -110,6 +118,15 @@ public CompletableFuture asyncGetQueue(NamespaceName namespaceName, S removeQueueFuture(namespaceName, queueName); return; } + } else { + amqpQueue.startMessageExpireChecker() + .whenComplete((__, t) -> { + if (t != null) { + log.error("Failed to start message expire checker queue [{}]", queueName, t); + } + queueCompletableFuture.complete(amqpQueue); + }); + return; } queueCompletableFuture.complete(amqpQueue); } @@ -119,11 +136,146 @@ public CompletableFuture asyncGetQueue(NamespaceName namespaceName, S return queueCompletableFuture; } + // TODO need to improve + public CompletableFuture asyncGetQueue(NamespaceName namespaceName, String queueName, + boolean passive, + boolean durable, + boolean exclusive, + boolean autoDelete, + boolean nowait, + Map arguments) { + CompletableFuture queueCompletableFuture = new CompletableFuture<>(); + if (namespaceName == null || StringUtils.isEmpty(queueName)) { + log.error("[{}][{}] Parameter error, namespaceName or queueName is empty.", namespaceName, queueName); + queueCompletableFuture.completeExceptionally( + new IllegalArgumentException("NamespaceName or queueName is empty")); + return queueCompletableFuture; + } + if (pulsarService.getState() != PulsarService.State.Started) { + log.error("Pulsar service not started."); + queueCompletableFuture.completeExceptionally(new PulsarServerException("PulsarService not start")); + return queueCompletableFuture; + } + String topicName = PersistentQueue.getQueueTopicName(namespaceName, queueName); + + Map initProperties = new HashMap<>(); + + if (!passive) { + try { + initProperties.putAll(QueueUtil.generateTopicProperties(queueName, durable, + autoDelete, passive, arguments)); + } catch (Exception e) { + log.error("Failed to generate topic properties for exchange {} in vhost {}.", + queueName, namespaceName, e); + queueCompletableFuture.completeExceptionally(e); + removeQueueFuture(namespaceName, queueName); + return queueCompletableFuture; + } + } + queueMap.putIfAbsent(namespaceName, new ConcurrentHashMap<>()); + CompletableFuture existingAmqpQueueFuture = queueMap.get(namespaceName). + putIfAbsent(queueName, queueCompletableFuture); + if (existingAmqpQueueFuture != null) { + return existingAmqpQueueFuture.thenCompose(amqpQueue -> { + if (amqpQueue instanceof PersistentQueue persistentQueue) { + if (!passive) { + if (!queueDeclareCheck( + queueCompletableFuture, namespaceName.getLocalName(), + queueName, initProperties, persistentQueue.getProperties())) { + return queueCompletableFuture; + } + } + } + return existingAmqpQueueFuture; + }); + } + + CompletableFuture topicCompletableFuture = + amqpTopicManager.getTopic(topicName, !passive, initProperties); + topicCompletableFuture.whenComplete((topic, throwable) -> { + if (throwable != null) { + log.error("[{}][{}] Failed to get queue topic.", namespaceName, queueName, throwable); + queueCompletableFuture.completeExceptionally(throwable); + removeQueueFuture(namespaceName, queueName); + } else { + if (null == topic) { + log.warn("[{}][{}] Queue topic did not exist.", namespaceName, queueName); + queueCompletableFuture.completeExceptionally( + new AoPServiceRuntimeException.NoSuchQueueException( + "Queue [" + queueName + "] not created")); + removeQueueFuture(namespaceName, queueName); + } else { + // recover metadata if existed + PersistentTopic persistentTopic = (PersistentTopic) topic; + Map properties = persistentTopic.getManagedLedger().getProperties(); + + if (!passive && !queueDeclareCheck( + queueCompletableFuture, namespaceName.getLocalName(), + queueName, initProperties, properties)) { + removeQueueFuture(namespaceName, queueName); + return; + } + // TODO: reset connectionId, exclusive and autoDelete + PersistentQueue amqpQueue = new PersistentQueue(queueName, persistentTopic, + 0, false, false, properties); + if (!config.isAmqpMultiBundleEnable()) { + try { + amqpQueue.recoverRoutersFromQueueProperties(properties, exchangeContainer, + namespaceName); + } catch (Exception e) { + log.error("[{}][{}] Failed to recover routers for queue from properties.", + namespaceName, queueName, e); + queueCompletableFuture.completeExceptionally(e); + removeQueueFuture(namespaceName, queueName); + return; + } + } else { + amqpQueue.startMessageExpireChecker() + .whenComplete((__, t) -> { + if (t != null) { + log.error("Failed to start message expire checker queue [{}]", queueName, t); + } + queueCompletableFuture.complete(amqpQueue); + }); + return; + } + queueCompletableFuture.complete(amqpQueue); + } + } + }).exceptionally(throwable -> { + log.error("[{}][{}] Failed to create persistent queue.", namespaceName, queueName, throwable); + queueCompletableFuture.completeExceptionally(throwable); + removeQueueFuture(namespaceName, queueName); + return null; + }); + return queueCompletableFuture; + } + + private boolean queueDeclareCheck(CompletableFuture queueFuture, String vhost, + String queueName, Map arguments, Map properties) { + + String replyTextFormat = "PRECONDITION_FAILED - inequivalent arg '%s' for queue '" + queueName + "' in " + + "vhost '" + vhost + "': received '%s' but current is '%s'"; + if (properties == null) { + queueFuture.completeExceptionally(new AoPException(ErrorCodes.IN_USE, + String.format(replyTextFormat, "queueInfo", arguments, properties), true, true)); + return false; + } + String oldArgs = properties.get("ARGUMENTS"); + String newArgs = arguments.get("ARGUMENTS"); + if (!QueueUtil.covertStringValueAsObjectMap(oldArgs).equals(QueueUtil.covertStringValueAsObjectMap(newArgs))) { + queueFuture.completeExceptionally(new AoPException(ErrorCodes.IN_USE, + String.format(replyTextFormat, "arguments", newArgs, oldArgs), true, true)); + return false; + } + return true; + } + /** * Delete the queue by namespace and exchange name. * * @param namespaceName namespace name in pulsar - * @param queueName name of queue + * @param queueName name of queue */ public void deleteQueue(NamespaceName namespaceName, String queueName) { if (StringUtils.isEmpty(queueName)) { @@ -134,7 +286,7 @@ public void deleteQueue(NamespaceName namespaceName, String queueName) { private void removeQueueFuture(NamespaceName namespaceName, String queueName) { if (queueMap.containsKey(namespaceName)) { - queueMap.get(namespaceName).remove(queueName); + queueMap.get(namespaceName).remove(queueName).thenAccept(AmqpQueue::close); } } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/QueueService.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/QueueService.java index 7c0fb9ce..5363f727 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/QueueService.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/QueueService.java @@ -14,6 +14,7 @@ package io.streamnative.pulsar.handlers.amqp; +import io.streamnative.pulsar.handlers.amqp.admin.model.BindingParams; import java.util.Map; import java.util.concurrent.CompletableFuture; import org.apache.pulsar.common.naming.NamespaceName; @@ -61,7 +62,7 @@ CompletableFuture queueDelete(NamespaceName namespaceName, String queue, b * @param argumentsTable other properties (binding parameters) */ CompletableFuture queueBind(NamespaceName namespaceName, String queue, String exchange, String bindingKey, - boolean nowait, FieldTable argumentsTable, long connectionId); + boolean nowait, FieldTable argumentsTable, long connectionId); /** * Unbinds a queue from an exchange. @@ -72,7 +73,7 @@ CompletableFuture queueBind(NamespaceName namespaceName, String queue, Str * @param arguments other properties (binding parameters) */ CompletableFuture queueUnbind(NamespaceName namespaceName, String queue, String exchange, String bindingKey, - FieldTable arguments, long connectionId); + FieldTable arguments, long connectionId); /** * Purges the contents of the given queue. @@ -84,4 +85,10 @@ CompletableFuture queueUnbind(NamespaceName namespaceName, String queue, S CompletableFuture getQueue(NamespaceName namespaceName, String queue, boolean createIfMissing, long connectionId); + CompletableFuture getQueue(NamespaceName namespaceName, String queue, boolean createIfMissing, + long connectionId, boolean durable, boolean exclusive, boolean autoDelete, + boolean nowait, Map arguments); + + CompletableFuture queueBind(NamespaceName namespaceName, String queue, String exchange, BindingParams params); + CompletableFuture queueUnBind(NamespaceName namespaceName, String queue, String exchange, String params); } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/QueueServiceImpl.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/QueueServiceImpl.java index 2b8e0794..9699a95d 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/QueueServiceImpl.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/QueueServiceImpl.java @@ -15,20 +15,39 @@ package io.streamnative.pulsar.handlers.amqp; import static io.streamnative.pulsar.handlers.amqp.utils.ExceptionUtils.getAoPException; +import static io.streamnative.pulsar.handlers.amqp.utils.ExchangeUtil.JSON_MAPPER; import static io.streamnative.pulsar.handlers.amqp.utils.ExchangeUtil.getExchangeType; import static io.streamnative.pulsar.handlers.amqp.utils.ExchangeUtil.isBuildInExchange; import static io.streamnative.pulsar.handlers.amqp.utils.ExchangeUtil.isDefaultExchange; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.collect.Sets; +import io.streamnative.pulsar.handlers.amqp.admin.model.BindingParams; import io.streamnative.pulsar.handlers.amqp.common.exception.AoPException; +import io.streamnative.pulsar.handlers.amqp.common.exception.AoPServiceRuntimeException; +import io.streamnative.pulsar.handlers.amqp.impl.PersistentExchange; +import io.streamnative.pulsar.handlers.amqp.impl.PersistentQueue; +import io.streamnative.pulsar.handlers.amqp.utils.QueueUtil; import java.util.Collection; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.broker.service.Consumer; +import org.apache.pulsar.broker.service.Subscription; +import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.qpid.server.protocol.ErrorCodes; import org.apache.qpid.server.protocol.v0_8.AMQShortString; import org.apache.qpid.server.protocol.v0_8.FieldTable; @@ -40,11 +59,13 @@ public class QueueServiceImpl implements QueueService { private ExchangeContainer exchangeContainer; private QueueContainer queueContainer; + private AmqpTopicManager amqpTopicManager; public QueueServiceImpl(ExchangeContainer exchangeContainer, - QueueContainer queueContainer) { + QueueContainer queueContainer, AmqpTopicManager amqpTopicManager) { this.exchangeContainer = exchangeContainer; this.queueContainer = queueContainer; + this.amqpTopicManager = amqpTopicManager; } @Override @@ -58,21 +79,22 @@ public CompletableFuture queueDeclare(NamespaceName namespaceName, St finalQueue = AMQShortString.createAMQShortString(queue); } CompletableFuture future = new CompletableFuture<>(); - getQueue(namespaceName, finalQueue.toString(), !passive, connectionId) + getQueue(namespaceName, finalQueue.toString(), passive, connectionId, + durable, exclusive, autoDelete, nowait, arguments) .whenComplete((amqpQueue, throwable) -> { - if (throwable != null) { - log.error("Failed to get topic from queue container", throwable); - future.completeExceptionally(getAoPException(throwable, "Failed to get queue: " + finalQueue + ", " - + throwable.getMessage(), true, false)); - } else { - if (null == amqpQueue) { - future.completeExceptionally( - new AoPException(ErrorCodes.NOT_FOUND, "No such queue: " + finalQueue, true, false)); - } else { - future.complete(amqpQueue); - } - } - }); + if (throwable != null) { + log.error("Failed to get topic from queue container", throwable); + future.completeExceptionally(getAoPException(throwable, "Failed to get queue: " + finalQueue + ", " + + throwable.getMessage(), true, false)); + } else { + if (null == amqpQueue) { + future.completeExceptionally( + new AoPException(ErrorCodes.NOT_FOUND, "No such queue: " + finalQueue, true, false)); + } else { + future.complete(amqpQueue); + } + } + }); return future; } @@ -86,32 +108,50 @@ public CompletableFuture queueDelete(NamespaceName namespaceName, String q CompletableFuture future = new CompletableFuture<>(); getQueue(namespaceName, queue, false, connectionId) .whenComplete((amqpQueue, throwable) -> { - if (throwable != null) { - log.error("Failed to get topic from queue container", throwable); - future.completeExceptionally(getAoPException(throwable, "Failed to get queue: " - + queue + ", " + throwable.getMessage(), true, false)); - } else { - if (null == amqpQueue) { - future.completeExceptionally( - new AoPException(ErrorCodes.NOT_FOUND, "No such queue: " + queue, true, false)); - } else { - Collection routers = amqpQueue.getRouters(); - if (!CollectionUtils.isEmpty(routers)) { - for (AmqpMessageRouter router : routers) { - // TODO need to change to async way - amqpQueue.unbindExchange(router.getExchange()); + if (throwable != null) { + log.error("Failed to get topic from queue container", throwable); + future.completeExceptionally(getAoPException(throwable, "Failed to get queue: " + + queue + ", " + throwable.getMessage(), true, false)); + } else { + if (null == amqpQueue) { + future.completeExceptionally( + new AoPException(ErrorCodes.NOT_FOUND, "No such queue: " + queue, true, false)); + } else { + Topic topic = amqpQueue.getTopic(); + ConcurrentOpenHashMap subscriptions = topic.getSubscriptions(); + if (subscriptions != null) { + Subscription subscription = subscriptions.get(PersistentQueue.DEFAULT_SUBSCRIPTION); + if (subscription != null) { + if (ifUnused && CollectionUtils.isNotEmpty(subscription.getConsumers())) { + future.completeExceptionally(new AoPException(ErrorCodes.INTERNAL_ERROR, "Failed to delete queue: " + + queue + ", Queue has active consumers", true, false)); + return; + } + if (ifEmpty && subscription.getNumberOfEntriesInBacklog(false) > 0) { + future.completeExceptionally(new AoPException(ErrorCodes.INTERNAL_ERROR, "Failed to delete queue: " + + queue + ", Queue has message", true, false)); + return; + } + } + } + Collection routers = amqpQueue.getRouters(); + if (!CollectionUtils.isEmpty(routers)) { + for (AmqpMessageRouter router : routers) { + // TODO need to change to async way + amqpQueue.unbindExchange(router.getExchange()); + } + } + amqpQueue.close(); + amqpQueue.getTopic().deleteForcefully().thenAccept(__ -> { + queueContainer.deleteQueue(namespaceName, amqpQueue.getName()); + future.complete(null); + }).exceptionally(t -> { + future.completeExceptionally(t); + return null; + }); } } - amqpQueue.getTopic().delete().thenAccept(__ -> { - queueContainer.deleteQueue(namespaceName, amqpQueue.getName()); - future.complete(null); - }).exceptionally(t -> { - future.completeExceptionally(t); - return null; - }); - } - } - }); + }); return future; } @@ -161,45 +201,45 @@ public CompletableFuture queueUnbind(NamespaceName namespaceName, String q CompletableFuture future = new CompletableFuture<>(); getQueue(namespaceName, queue, false, connectionId) .whenComplete((amqpQueue, throwable) -> { - if (throwable != null) { - log.error("Failed to get topic from queue container", throwable); - future.completeExceptionally(getAoPException(throwable, - "Failed to get queue: " + throwable.getMessage(), true, false)); - } else { - if (amqpQueue == null) { - future.completeExceptionally(new AoPException(ErrorCodes.NOT_FOUND, - "No such queue: '" + queue + "'", true, false)); - return; - } - String exchangeName; - if (isDefaultExchange(exchange)) { - exchangeName = AbstractAmqpExchange.DEFAULT_EXCHANGE_DURABLE; - } else { - exchangeName = exchange; - } - CompletableFuture amqpExchangeCompletableFuture = - exchangeContainer.asyncGetExchange(namespaceName, exchangeName, false, null); - amqpExchangeCompletableFuture.whenComplete((amqpExchange, exThrowable) -> { - if (exThrowable != null) { - log.error("Failed to get topic from exchange container", exThrowable); - future.completeExceptionally(getAoPException(exThrowable, - "Failed to get exchange: " + exThrowable.getMessage(), true, false)); + if (throwable != null) { + log.error("Failed to get topic from queue container", throwable); + future.completeExceptionally(getAoPException(throwable, + "Failed to get queue: " + throwable.getMessage(), true, false)); } else { - try { - amqpQueue.unbindExchange(amqpExchange); - if (amqpExchange.getAutoDelete() && (amqpExchange.getQueueSize() == 0)) { - exchangeContainer.deleteExchange(namespaceName, exchangeName); - amqpExchange.getTopic().delete().get(); - } - future.complete(null); - } catch (Exception e) { - future.completeExceptionally(getAoPException(e, - "Unbind failed:" + e.getMessage(), false, true)); + if (amqpQueue == null) { + future.completeExceptionally(new AoPException(ErrorCodes.NOT_FOUND, + "No such queue: '" + queue + "'", true, false)); + return; + } + String exchangeName; + if (isDefaultExchange(exchange)) { + exchangeName = AbstractAmqpExchange.DEFAULT_EXCHANGE_DURABLE; + } else { + exchangeName = exchange; } + CompletableFuture amqpExchangeCompletableFuture = + exchangeContainer.asyncGetExchange(namespaceName, exchangeName, false, null); + amqpExchangeCompletableFuture.whenComplete((amqpExchange, exThrowable) -> { + if (exThrowable != null) { + log.error("Failed to get topic from exchange container", exThrowable); + future.completeExceptionally(getAoPException(exThrowable, + "Failed to get exchange: " + exThrowable.getMessage(), true, false)); + } else { + try { + amqpQueue.unbindExchange(amqpExchange); + if (amqpExchange.getAutoDelete() && (amqpExchange.getQueueSize() == 0)) { + exchangeContainer.deleteExchange(namespaceName, exchangeName); + amqpExchange.getTopic().delete().get(); + } + future.complete(null); + } catch (Exception e) { + future.completeExceptionally(getAoPException(e, + "Unbind failed:" + e.getMessage(), false, true)); + } + } + }); } }); - } - }); return future; } @@ -211,7 +251,7 @@ public CompletableFuture queuePurge(NamespaceName namespaceName, String qu } private CompletableFuture bind(NamespaceName namespaceName, String exchange, AmqpQueue amqpQueue, - String bindingKey, Map arguments) { + String bindingKey, Map arguments) { String exchangeName = isDefaultExchange(exchange) ? AbstractAmqpExchange.DEFAULT_EXCHANGE_DURABLE : exchange; if (exchangeName.equals(AbstractAmqpExchange.DEFAULT_EXCHANGE_DURABLE)) { @@ -228,27 +268,27 @@ private CompletableFuture bind(NamespaceName namespaceName, String exchang CompletableFuture future = new CompletableFuture<>(); exchangeContainer.asyncGetExchange(namespaceName, exchangeName, createIfMissing, exchangeType) .whenComplete((amqpExchange, throwable) -> { - if (throwable != null) { - log.error("Failed to get topic from exchange container", throwable); - future.completeExceptionally(new AoPException(ErrorCodes.INTERNAL_ERROR, "Failed to get exchange: " - + throwable.getMessage(), true, false)); - return; - } - AmqpMessageRouter messageRouter = AbstractAmqpMessageRouter.generateRouter(amqpExchange.getType()); - if (messageRouter == null) { - future.completeExceptionally(new AoPException(ErrorCodes.INTERNAL_ERROR, "Unsupported router type!", - false, true)); - return; - } - try { - amqpQueue.bindExchange(amqpExchange, messageRouter, bindingKey, arguments); - future.complete(null); - } catch (Exception e) { - log.warn("Failed to bind queue[{}] with exchange[{}].", amqpQueue.getName(), exchange, e); - future.completeExceptionally(new AoPException(ErrorCodes.INTERNAL_ERROR, - "Catch a PulsarAdminException: " + e.getMessage() + ". ", false, true)); - } - }); + if (throwable != null) { + log.error("Failed to get topic from exchange container", throwable); + future.completeExceptionally(new AoPException(ErrorCodes.INTERNAL_ERROR, "Failed to get exchange: " + + throwable.getMessage(), true, false)); + return; + } + AmqpMessageRouter messageRouter = AbstractAmqpMessageRouter.generateRouter(amqpExchange.getType()); + if (messageRouter == null) { + future.completeExceptionally(new AoPException(ErrorCodes.INTERNAL_ERROR, "Unsupported router type!", + false, true)); + return; + } + try { + amqpQueue.bindExchange(amqpExchange, messageRouter, bindingKey, arguments); + future.complete(null); + } catch (Exception e) { + log.warn("Failed to bind queue[{}] with exchange[{}].", amqpQueue.getName(), exchange, e); + future.completeExceptionally(new AoPException(ErrorCodes.INTERNAL_ERROR, + "Catch a PulsarAdminException: " + e.getMessage() + ". ", false, true)); + } + }); return future; } @@ -272,5 +312,129 @@ public CompletableFuture getQueue(NamespaceName namespaceName, String }); return future; } + @Override + public CompletableFuture getQueue(NamespaceName namespaceName, String queueName, boolean passive, + long connectionId, boolean durable, boolean exclusive, boolean autoDelete, + boolean nowait, Map arguments) { + CompletableFuture future = new CompletableFuture<>(); + queueContainer.asyncGetQueue(namespaceName, queueName, passive, durable, exclusive, autoDelete, nowait, + arguments).whenComplete((queue, throwable) -> { + if (throwable != null) { + future.completeExceptionally(new AoPException(ErrorCodes.INTERNAL_ERROR, throwable.getMessage(), false, true)); + return; + } + if (queue != null && queue.isExclusive() && queue.getConnectionId() != connectionId) { + future.completeExceptionally(new AoPException(ErrorCodes.ALREADY_EXISTS, + "cannot obtain exclusive access to locked queue '" + queue + "' in vhost '" + + namespaceName.getLocalName() + "'", false, true)); + return; + } + future.complete(queue); + }); + return future; + } + + /** + * Query the list of exchanges bound to the queue + * + * @param namespaceName + * @param queue + * @param exchange + * @param params + * @return + */ + @Override + public CompletableFuture queueBind(NamespaceName namespaceName, String queue, String exchange, + BindingParams params) { + String topicName = PersistentQueue.getQueueTopicName(namespaceName, queue); + return amqpTopicManager.getTopic(topicName, false, null) + .thenCompose(topic -> { + if (topic == null) { + throw new AoPServiceRuntimeException.NoSuchQueueException("Queue [" + queue + "] not created"); + } + PersistentTopic persistentTopic = (PersistentTopic) topic; + Set bindings = Sets.newHashSet(); + String bindingsJson; + try { + if (persistentTopic.getManagedLedger().getProperties().containsKey("BINDINGS")) { + List amqpQueueProperties = JSON_MAPPER.readValue( + persistentTopic.getManagedLedger().getProperties().get("BINDINGS"), + new TypeReference>() { + }); + bindings.addAll(amqpQueueProperties); + } + bindings.add(new PersistentExchange.Binding(queue, "queue", params.getRoutingKey(), exchange, + params.getArguments())); + bindingsJson = JSON_MAPPER.writeValueAsString(bindings); + } catch (JsonProcessingException e) { + log.error("Failed to bind queue {} to exchange {}", queue, exchange, e); + return FutureUtil.failedFuture(e); + } + CompletableFuture future = new CompletableFuture<>(); + persistentTopic.getManagedLedger().asyncSetProperty("BINDINGS", bindingsJson, + new AsyncCallbacks.UpdatePropertiesCallback() { + @Override + public void updatePropertiesComplete(Map properties, Object ctx) { + future.complete(null); + } + + @Override + public void updatePropertiesFailed(ManagedLedgerException exception, Object ctx) { + log.error("Failed to save binding metadata for bind operation.", exception); + future.completeExceptionally(exception); + } + }, null); + return future; + }); + } + + @Override + public CompletableFuture queueUnBind(NamespaceName namespaceName, String queue, String exchange, + String propsKey) { + String topicName = PersistentQueue.getQueueTopicName(namespaceName, queue); + return amqpTopicManager.getTopic(topicName, false, null) + .thenCompose(topic -> { + if (topic == null) { + throw new AoPServiceRuntimeException.NoSuchQueueException("Queue [" + queue + "] not created"); + } + PersistentTopic persistentTopic = (PersistentTopic) topic; + Set bindings = Sets.newHashSet(); + String bindingsJson; + try { + if (persistentTopic.getManagedLedger().getProperties().containsKey("BINDINGS")) { + List amqpQueueProperties = JSON_MAPPER.readValue( + persistentTopic.getManagedLedger().getProperties().get("BINDINGS"), + new TypeReference>() { + }); + bindings.addAll(amqpQueueProperties); + } + bindings.removeIf(binding -> exchange.equals(binding.getSource()) + && queue.equals(binding.getDes()) + && propsKey.equals(binding.getKey())); + bindingsJson = JSON_MAPPER.writeValueAsString(bindings); + } catch (JsonProcessingException e) { + log.error("Failed to bind queue {} to exchange {}", queue, exchange, e); + return FutureUtil.failedFuture(e); + } + CompletableFuture future = new CompletableFuture<>(); + persistentTopic.getManagedLedger().asyncSetProperty("BINDINGS", bindingsJson, + new AsyncCallbacks.UpdatePropertiesCallback() { + @Override + public void updatePropertiesComplete(Map properties, Object ctx) { + future.complete(null); + } + + @Override + public void updatePropertiesFailed(ManagedLedgerException exception, Object ctx) { + log.error("Failed to save binding metadata for bind operation.", exception); + future.completeExceptionally(exception); + } + }, null); + return future; + }).exceptionally(throwable -> { + log.error("Failed to save binding metadata for bind operation.", throwable); + return null; + }); + } } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/UnacknowledgedMessageMap.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/UnacknowledgedMessageMap.java index 3acd7cc9..88124b8a 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/UnacknowledgedMessageMap.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/UnacknowledgedMessageMap.java @@ -36,6 +36,7 @@ public class UnacknowledgedMessageMap { public interface MessageProcessor { void messageAck(Position position); void requeue(List positions); + default void discardMessage(List positions){} } /** diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/AmqpAdmin.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/AmqpAdmin.java index 2ffca9b0..f6a0acb1 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/AmqpAdmin.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/AmqpAdmin.java @@ -3,7 +3,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -14,11 +14,17 @@ package io.streamnative.pulsar.handlers.amqp.admin; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; import io.streamnative.pulsar.handlers.amqp.admin.model.BindingParams; import io.streamnative.pulsar.handlers.amqp.admin.model.ExchangeDeclareParams; import io.streamnative.pulsar.handlers.amqp.admin.model.QueueDeclareParams; +import io.streamnative.pulsar.handlers.amqp.admin.model.rabbitmq.QueueBinds; +import io.streamnative.pulsar.handlers.amqp.impl.PersistentExchange; import io.streamnative.pulsar.handlers.amqp.utils.HttpUtil; import io.streamnative.pulsar.handlers.amqp.utils.JsonUtil; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import org.apache.pulsar.common.naming.NamespaceName; @@ -33,51 +39,112 @@ public AmqpAdmin(String host, int port) { this.baseUrl = "http://" + host + ":" + port + "/api"; } - public CompletableFuture exchangeDeclare(String vhost, + public CompletableFuture exchangeDeclare(NamespaceName namespaceName, String exchange, ExchangeDeclareParams exchangeDeclareParams) { - NamespaceName namespaceName = NamespaceName.get(vhost); String url = String.format("%s/exchanges/%s/%s", baseUrl, namespaceName.getLocalName(), exchange); try { - return HttpUtil.putAsync(url, JsonUtil.toMap(exchangeDeclareParams)); + return HttpUtil.putAsync(url, JsonUtil.toMap(exchangeDeclareParams), + Map.of("tenant", namespaceName.getTenant())); } catch (JsonProcessingException e) { - return CompletableFuture.failedFuture(e); + return CompletableFuture.failedFuture(e); } } - public CompletableFuture queueDeclare(String vhost, + public CompletableFuture queueDeclare(NamespaceName namespaceName, String queue, QueueDeclareParams queueDeclareParams) { - NamespaceName namespaceName = NamespaceName.get(vhost); String url = String.format("%s/queues/%s/%s", baseUrl, namespaceName.getLocalName(), queue); try { - return HttpUtil.putAsync(url, JsonUtil.toMap(queueDeclareParams)); + return HttpUtil.putAsync(url, JsonUtil.toMap(queueDeclareParams), + Map.of("tenant", namespaceName.getTenant())); } catch (JsonProcessingException e) { return CompletableFuture.failedFuture(e); } } - public CompletableFuture queueBind(String vhost, + public CompletableFuture queueDelete(NamespaceName namespaceName, String queue, Map params) { + String url = String.format("%s/queues/%s/%s", baseUrl, namespaceName.getLocalName(), queue); + return HttpUtil.deleteAsync(url, params, Map.of("tenant", namespaceName.getTenant())); + } + + public CompletableFuture queuePurge(NamespaceName namespaceName, String queue, Map params) { + String url = String.format("%s/queues/%s/%s/contents", baseUrl, namespaceName.getLocalName(), queue); + return HttpUtil.deleteAsync(url, params, Map.of("tenant", namespaceName.getTenant())); + } + + public CompletableFuture exchangeDelete(NamespaceName namespaceName, String exchange, Map params) { + String url = String.format("%s/exchanges/%s/%s", baseUrl, namespaceName.getLocalName(), exchange); + return HttpUtil.deleteAsync(url, params, Map.of("tenant", namespaceName.getTenant())); + } + + public void startExpirationDetection(NamespaceName namespaceName, String queue) { + String url = String.format("%s/queues/%s/%s/startExpirationDetection", baseUrl, namespaceName.getLocalName(), queue); + HttpUtil.putAsync(url, new HashMap<>(1), Map.of("tenant", namespaceName.getTenant())); + } + + public CompletableFuture queueBind(NamespaceName namespaceName, String exchange, String queue, BindingParams bindingParams) { - NamespaceName namespaceName = NamespaceName.get(vhost); String url = String.format("%s/bindings/%s/e/%s/q/%s", baseUrl, namespaceName.getLocalName(), exchange, queue); try { - return HttpUtil.postAsync(url, JsonUtil.toMap(bindingParams)); + return HttpUtil.postAsync(url, JsonUtil.toMap(bindingParams), Map.of("tenant", namespaceName.getTenant())); + } catch (JsonProcessingException e) { + return CompletableFuture.failedFuture(e); + } + } + public CompletableFuture loadExchange(NamespaceName namespaceName, String ex) { + String url = String.format("%s/exchanges/%s/%s/loadExchange", baseUrl, namespaceName.getLocalName(), ex); + return HttpUtil.getAsync(url, Map.of("tenant", namespaceName.getTenant())); + } + public CompletableFuture loadQueue(NamespaceName namespaceName, String queue) { + String url = String.format("%s/queues/%s/%s/loadQueue", baseUrl, namespaceName.getLocalName(), queue); + return HttpUtil.getAsync(url, Map.of("tenant", namespaceName.getTenant())); + } + public CompletableFuture loadAllExchangeByVhost(NamespaceName namespaceName) { + String url = String.format("%s/exchanges/%s/loadAllExchangeByVhost", baseUrl, namespaceName.getLocalName()); + return HttpUtil.getAsync(url, Map.of("tenant", namespaceName.getTenant())); + } + public CompletableFuture loadAllVhostForExchange(String tenant) { + String url = String.format("%s/vhosts/loadAllVhostForExchange", baseUrl); + return HttpUtil.getAsync(url, Map.of("tenant",tenant)); + } + + public CompletableFuture> getQueueBindings(NamespaceName namespaceName, String queue) { + String url = String.format("%s/queues/%s/%s/bindings", baseUrl, namespaceName.getLocalName(), queue); + return HttpUtil.getAsync(url, Map.of("tenant", namespaceName.getTenant()), new TypeReference>() { + }); + } + + public CompletableFuture queueBindExchange(NamespaceName namespaceName, + String exchange, + String queue, + BindingParams bindingParams) { + String url = String.format("%s/queueBindExchange/%s/e/%s/q/%s", baseUrl, namespaceName.getLocalName(), exchange, + queue); + try { + return HttpUtil.postAsync(url, JsonUtil.toMap(bindingParams), Map.of("tenant", namespaceName.getTenant())); } catch (JsonProcessingException e) { return CompletableFuture.failedFuture(e); } } - public CompletableFuture queueUnbind(String vhost, + public CompletableFuture queueUnBindExchange(NamespaceName namespaceName, + String exchange, + String queue, + String props) { + String url = String.format("%s/queueUnBindExchange/%s/e/%s/q/%s/unbind", + baseUrl, namespaceName.getLocalName(), exchange, queue); + return HttpUtil.deleteAsync(url, Map.of("properties_key", props), Map.of("tenant", namespaceName.getTenant())); + } + + public CompletableFuture queueUnbind(NamespaceName namespaceName, String exchange, String queue, String props) { - NamespaceName namespaceName = NamespaceName.get(vhost); - String url = String.format("%s/bindings/%s/e/%s/q/%s/%s", - baseUrl, namespaceName.getLocalName(), exchange, queue, props); - return HttpUtil.deleteAsync(url, null); + String url = String.format("%s/bindings/%s/e/%s/q/%s/unbind", + baseUrl, namespaceName.getLocalName(), exchange, queue); + return HttpUtil.deleteAsync(url, Map.of("properties_key", props), Map.of("tenant", namespaceName.getTenant())); } - } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/Bindings.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/Bindings.java index 9a183aeb..c7c86233 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/Bindings.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/Bindings.java @@ -15,6 +15,7 @@ import io.streamnative.pulsar.handlers.amqp.admin.impl.BindingBase; import io.streamnative.pulsar.handlers.amqp.admin.model.BindingParams; +import io.streamnative.pulsar.handlers.amqp.admin.model.QueueUnBindingParams; import io.streamnative.pulsar.handlers.amqp.impl.PersistentExchange; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; @@ -57,16 +58,18 @@ public void getList(@Suspended final AsyncResponse response, @POST @Path("/{vhost}/e/{exchange}/q/{queue}") public void queueBind(@Suspended final AsyncResponse response, - @PathParam("vhost") String vhost, - @PathParam("exchange") String exchange, - @PathParam("queue") String queue, - BindingParams params, + @PathParam("vhost") String vhost, + @PathParam("exchange") String exchange, + @PathParam("queue") String queue, + BindingParams params, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + NamespaceName namespaceName = getNamespaceName(vhost); TopicName topicName = TopicName.get(TopicDomain.persistent.toString(), - NamespaceName.get("public", vhost), PersistentExchange.TOPIC_PREFIX + exchange); + namespaceName, PersistentExchange.TOPIC_PREFIX + exchange); validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(__ -> queueBindAsync(vhost, exchange, queue, params)) + .thenCompose(__ -> queueBindAsync(namespaceName, exchange, queue, params)) .thenAccept(__ -> response.resume(Response.noContent().build())) + .thenComposeAsync(__ -> amqpAdmin().queueBindExchange(namespaceName, exchange, queue, params)) .exceptionally(t -> { if (!isRedirectException(t)) { log.error("Failed to bind queue {} to exchange {} with key {} in vhost {}", @@ -80,10 +83,10 @@ public void queueBind(@Suspended final AsyncResponse response, @GET @Path("/{vhost}/e/{exchange}/q/{queue}/{props}") public void getQueueBinding(@Suspended final AsyncResponse response, - @PathParam("vhost") String vhost, - @PathParam("exchange") String exchange, - @PathParam("queue") String queue, - @PathParam("props") String propsKey) { + @PathParam("vhost") String vhost, + @PathParam("exchange") String exchange, + @PathParam("queue") String queue, + @PathParam("props") String propsKey) { getBindingsByPropsKeyAsync(vhost, exchange, queue, propsKey) .thenAccept(response::resume) .exceptionally(t -> { @@ -95,21 +98,25 @@ public void getQueueBinding(@Suspended final AsyncResponse response, } @DELETE - @Path("/{vhost}/e/{exchange}/q/{queue}/{props}") - public void queueUnbind(@Suspended final AsyncResponse response, - @PathParam("vhost") String vhost, - @PathParam("exchange") String exchange, - @PathParam("queue") String queue, - @PathParam("props") String propsKey, - @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + @Path("/{vhost}/e/{exchange}/q/{queue}/unbind") + public void queuUnbind(@Suspended final AsyncResponse response, + @PathParam("vhost") String vhost, + @PathParam("exchange") String exchange, + @PathParam("queue") String queue, + QueueUnBindingParams params, + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + NamespaceName namespaceName = getNamespaceName(vhost); TopicName topicName = TopicName.get(TopicDomain.persistent.toString(), - NamespaceName.get("public", vhost), PersistentExchange.TOPIC_PREFIX + exchange); + namespaceName, PersistentExchange.TOPIC_PREFIX + exchange); validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(__ -> queueUnbindAsync(vhost, exchange, queue, propsKey)) + .thenCompose(__ -> queueUnbindAsync(namespaceName, exchange, queue, params.getProperties_key())) + .thenCompose(__ -> amqpAdmin().queueUnBindExchange(namespaceName, exchange, queue, params.getProperties_key())) .thenAccept(__ -> response.resume(Response.noContent().build())) .exceptionally(t -> { - log.error("Failed to unbind queue {} to exchange {} with key {} in vhost {}", - queue, exchange, propsKey, vhost, t); + if (!isRedirectException(t)) { + log.error("Failed to unbind queue {} to exchange {} with key {} in vhost {}", + queue, exchange, params.getProperties_key(), vhost, t); + } resumeAsyncResponseExceptionally(response, t); return null; }); diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/Exchanges.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/Exchanges.java index b89a289f..da4e75ee 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/Exchanges.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/Exchanges.java @@ -3,7 +3,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,6 +15,7 @@ import io.streamnative.pulsar.handlers.amqp.admin.impl.ExchangeBase; import io.streamnative.pulsar.handlers.amqp.admin.model.ExchangeDeclareParams; +import io.streamnative.pulsar.handlers.amqp.admin.model.ExchangeDeleteParams; import io.streamnative.pulsar.handlers.amqp.impl.PersistentExchange; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; @@ -69,7 +70,10 @@ public void getListByVhost(@Suspended final AsyncResponse response, @Path("/{vhost}/{exchange}") public void getExchange(@Suspended final AsyncResponse response, @PathParam("vhost") String vhost, - @PathParam("exchange") String exchange) { + @PathParam("exchange") String exchange, + @QueryParam("msg_rates_age") int age, + @QueryParam("msg_rates_incr") int incr, + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { getExchangeBeanAsync(vhost, exchange) .thenAccept(response::resume) .exceptionally(t -> { @@ -80,6 +84,27 @@ public void getExchange(@Suspended final AsyncResponse response, }); } + @GET + @Path("/{vhost}/{exchange}/loadExchange") + public void loadExchange(@Suspended final AsyncResponse response, + @PathParam("vhost") String vhost, + @PathParam("exchange") String exchange, + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + TopicName topicName = TopicName.get(TopicDomain.persistent.toString(), + getNamespaceName(vhost), PersistentExchange.TOPIC_PREFIX + exchange); + validateTopicOwnershipAsync(topicName, authoritative) + .thenCompose(__ -> loadExchangeAsync(vhost, exchange)) + .thenAccept(__ -> response.resume(Response.noContent().build())) + .exceptionally(t -> { + if (!isRedirectException(t)) { + log.error("Failed to load exchange {} for tenant {} belong to vhost {}", + exchange, tenant, vhost, t); + } + resumeAsyncResponseExceptionally(response, t); + return null; + }); + } + @PUT @Path("/{vhost}/{exchange}") public void declareExchange(@Suspended final AsyncResponse response, @@ -87,10 +112,11 @@ public void declareExchange(@Suspended final AsyncResponse response, @PathParam("exchange") String exchange, ExchangeDeclareParams params, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + NamespaceName namespaceName = getNamespaceName(vhost); TopicName topicName = TopicName.get(TopicDomain.persistent.toString(), - NamespaceName.get(tenant, vhost), PersistentExchange.TOPIC_PREFIX + exchange); + namespaceName, PersistentExchange.TOPIC_PREFIX + exchange); validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(__ -> declareExchange(vhost, exchange, params)) + .thenCompose(__ -> declareExchange(namespaceName, exchange, params)) .thenAccept(__ -> response.resume(Response.noContent().build())) .exceptionally(t -> { if (!isRedirectException(t)) { @@ -104,18 +130,26 @@ public void declareExchange(@Suspended final AsyncResponse response, @DELETE @Path("/{vhost}/{exchange}") - public void declareExchange(@Suspended final AsyncResponse response, - @PathParam("vhost") String vhost, - @PathParam("exchange") String exchange, - @QueryParam("if-unused") boolean ifUnused) { - deleteExchange(vhost, exchange, ifUnused) + public void deleteExchange(@Suspended final AsyncResponse response, + @PathParam("vhost") String vhost, + @PathParam("exchange") String exchange, + ExchangeDeleteParams params, + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + NamespaceName namespaceName = getNamespaceName(vhost); + TopicName topicName = TopicName.get(TopicDomain.persistent.toString(), + namespaceName, PersistentExchange.TOPIC_PREFIX + exchange); + validateTopicOwnershipAsync(topicName, authoritative) + .thenCompose(__ -> deleteExchange(namespaceName, exchange, params.isIfUnused())) .thenAccept(__ -> { - log.info("Success delete exchange {} in vhost {}, ifUnused is {}", exchange, vhost, ifUnused); + log.info("Success delete exchange {} in vhost {}, ifUnused is {}", exchange, vhost, + params.isIfUnused()); response.resume(Response.noContent().build()); }) .exceptionally(t -> { - log.error("Failed to delete exchange {} for tenant {} belong to vhost {}", - exchange, tenant, vhost, t); + if (!isRedirectException(t)) { + log.error("Failed to delete exchange {} for tenant {} belong to vhost {}", + exchange, tenant, vhost, t); + } resumeAsyncResponseExceptionally(response, t); return null; }); diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/Queues.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/Queues.java index 16d1c8b8..bf69390a 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/Queues.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/Queues.java @@ -3,7 +3,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -14,11 +14,16 @@ package io.streamnative.pulsar.handlers.amqp.admin; import io.streamnative.pulsar.handlers.amqp.admin.impl.QueueBase; +import io.streamnative.pulsar.handlers.amqp.admin.model.BindingParams; import io.streamnative.pulsar.handlers.amqp.admin.model.QueueDeclareParams; +import io.streamnative.pulsar.handlers.amqp.admin.model.QueueDeleteParams; +import io.streamnative.pulsar.handlers.amqp.admin.model.QueueUnBindingParams; +import io.streamnative.pulsar.handlers.amqp.impl.PersistentExchange; import io.streamnative.pulsar.handlers.amqp.impl.PersistentQueue; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; +import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -66,11 +71,51 @@ public void getListByVhost(@Suspended final AsyncResponse response, } @GET - @Path("/{vhost}/{queue}") - public void getQueue(@Suspended final AsyncResponse response, - @PathParam("vhost") String vhost, - @PathParam("queue") String queue) { - getQueueBeanAsync(vhost, queue) + @Path("/{vhost}/{queue}/loadQueue") + public void loadQueue(@Suspended final AsyncResponse response, + @PathParam("vhost") String vhost, + @PathParam("queue") String queue, + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + TopicName topicName = TopicName.get(TopicDomain.persistent.toString(), + getNamespaceName(vhost), PersistentQueue.TOPIC_PREFIX + queue); + validateTopicOwnershipAsync(topicName, authoritative) + .thenCompose(__ -> loadQueueAsync(vhost, queue)) + .thenAccept(__ -> response.resume(Response.noContent().build())) + .exceptionally(t -> { + if (!isRedirectException(t)) { + log.error("Failed to declare queue {} {} in vhost {}", queue, tenant, vhost, t); + } + resumeAsyncResponseExceptionally(response, t); + return null; + }); + } + + @PUT + @Path("/{vhost}/{queue}/startExpirationDetection") + public void startExpirationDetection(@Suspended final AsyncResponse response, + @PathParam("vhost") String vhost, + @PathParam("queue") String queue, + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + TopicName topicName = TopicName.get(TopicDomain.persistent.toString(), + getNamespaceName(vhost), PersistentQueue.TOPIC_PREFIX + queue); + validateTopicOwnershipAsync(topicName, authoritative) + .thenCompose(__ -> startExpirationDetection(vhost, queue)) + .thenAccept(__ -> response.resume(Response.noContent().build())) + .exceptionally(t -> { + if (!isRedirectException(t)) { + log.error("Failed to declare queue {} {} in vhost {}", queue, tenant, vhost, t); + } + resumeAsyncResponseExceptionally(response, t); + return null; + }); + } + + @GET + @Path("/{vhost}/{queue}/bindings") + public void getQueueBindings(@Suspended final AsyncResponse response, + @PathParam("vhost") String vhost, + @PathParam("queue") String queue) { + getQueueBindings(getNamespaceName(vhost), queue) .thenAccept(response::resume) .exceptionally(t -> { log.error("Failed to get queue {} in vhost {}", queue, vhost, t); @@ -86,10 +131,11 @@ public void declareQueue(@Suspended final AsyncResponse response, @PathParam("queue") String queue, QueueDeclareParams params, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { - TopicName topicName = TopicName.get(TopicDomain.persistent.toString(), - NamespaceName.get("public", vhost), PersistentQueue.TOPIC_PREFIX + queue); + NamespaceName namespaceName = getNamespaceName(vhost); + TopicName topicName = + TopicName.get(TopicDomain.persistent.toString(), namespaceName, PersistentQueue.TOPIC_PREFIX + queue); validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(__ -> declareQueueAsync(vhost, queue, params)) + .thenCompose(__ -> declareQueueAsync(namespaceName, queue, params)) .thenAccept(__ -> response.resume(Response.noContent().build())) .exceptionally(t -> { if (!isRedirectException(t)) { @@ -100,21 +146,77 @@ public void declareQueue(@Suspended final AsyncResponse response, }); } + @POST + @Path("/{vhost}/e/{exchange}/q/{queue}") + public void queueBindings(@Suspended final AsyncResponse response, + @PathParam("vhost") String vhost, + @PathParam("exchange") String exchange, + @PathParam("queue") String queue, + BindingParams params, + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + NamespaceName namespaceName = getNamespaceName(vhost); + TopicName topicName = + TopicName.get(TopicDomain.persistent.toString(), namespaceName, + PersistentExchange.TOPIC_PREFIX + exchange); + validateTopicOwnershipAsync(topicName, authoritative) + .thenCompose(__ -> queueBindAsync(namespaceName, exchange, queue, params)) + .thenAccept(__ -> response.resume(Response.noContent().build())) + .thenRunAsync(() -> amqpAdmin().queueBindExchange(namespaceName, exchange, queue, params)) + .exceptionally(t -> { + if (!isRedirectException(t)) { + log.error("Failed to update queue {} {} in vhost {}", queue, tenant, vhost, t); + } + resumeAsyncResponseExceptionally(response, t); + return null; + }); + } + + @DELETE + @Path("/{vhost}/e/{exchange}/q/{queue}/unbind") + public void queueUnBindings(@Suspended final AsyncResponse response, + @PathParam("vhost") String vhost, + @PathParam("exchange") String exchange, + @PathParam("queue") String queue, + QueueUnBindingParams params, + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + NamespaceName namespaceName = getNamespaceName(vhost); + TopicName topicName = TopicName.get(TopicDomain.persistent.toString(), + namespaceName, PersistentExchange.TOPIC_PREFIX + exchange); + validateTopicOwnershipAsync(topicName, authoritative) + .thenCompose(__ -> queueUnbindAsync(namespaceName, exchange, queue, params.getProperties_key())) + .thenCompose(__ -> amqpAdmin().queueUnBindExchange(namespaceName, exchange, queue, params.getProperties_key())) + .thenAccept(__ -> response.resume(Response.noContent().build())) + .exceptionally(t -> { + if (!isRedirectException(t)) { + log.error("Failed to update queue {} {} in vhost {}", queue, tenant, vhost, t); + } + resumeAsyncResponseExceptionally(response, t); + return null; + }); + } + + @DELETE @Path("/{vhost}/{queue}") public void deleteQueue(@Suspended final AsyncResponse response, @PathParam("vhost") String vhost, @PathParam("queue") String queue, - @QueryParam("if-unused") boolean ifUnused, - @QueryParam("if-empty") boolean ifEmpty) { - deleteQueueAsync(vhost, queue, ifUnused, ifEmpty) + QueueDeleteParams params, + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + NamespaceName namespaceName = getNamespaceName(vhost); + TopicName topicName = TopicName.get(TopicDomain.persistent.toString(), + namespaceName, PersistentQueue.TOPIC_PREFIX + queue); + validateTopicOwnershipAsync(topicName, authoritative) + .thenCompose(__ -> deleteQueueAsync(namespaceName, queue, params.isIfUnused(), params.isIfEmpty())) .thenAccept(__ -> { log.info("Success delete queue {} in vhost {}, if-unused is {}, if-empty is {}", - queue, vhost, ifUnused, ifEmpty); + queue, vhost, params.isIfUnused(), params.isIfEmpty()); response.resume(Response.noContent().build()); }) .exceptionally(t -> { - log.error("Failed to delete queue {} in vhost {}", queue, vhost, t); + if (!isRedirectException(t)) { + log.error("Failed to delete queue {} in vhost {}", queue, vhost, t); + } resumeAsyncResponseExceptionally(response, t); return null; }); diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/impl/BaseResources.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/impl/BaseResources.java index ee1b2648..419b0c64 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/impl/BaseResources.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/impl/BaseResources.java @@ -3,7 +3,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,14 +18,26 @@ import io.streamnative.pulsar.handlers.amqp.ExchangeService; import io.streamnative.pulsar.handlers.amqp.QueueContainer; import io.streamnative.pulsar.handlers.amqp.QueueService; +import io.streamnative.pulsar.handlers.amqp.admin.AmqpAdmin; +import io.streamnative.pulsar.handlers.amqp.admin.model.BindingParams; import io.streamnative.pulsar.handlers.amqp.admin.model.VhostBean; +import io.streamnative.pulsar.handlers.amqp.common.exception.AoPServiceRuntimeException; +import java.math.BigDecimal; +import java.math.RoundingMode; import java.net.URI; import java.net.URL; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.stream.Stream; import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.HeaderParam; import javax.ws.rs.WebApplicationException; import javax.ws.rs.container.AsyncResponse; import javax.ws.rs.core.Context; @@ -33,17 +45,29 @@ import javax.ws.rs.core.UriBuilder; import javax.ws.rs.core.UriInfo; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; +import org.apache.bookkeeper.mledger.impl.MetaStore; +import org.apache.bookkeeper.mledger.proto.MLDataFormats; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.namespace.LookupOptions; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.resources.NamespaceResources; +import org.apache.pulsar.broker.resources.TenantResources; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.web.RestException; +import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.metadata.api.Stat; +import org.apache.qpid.server.protocol.v0_8.FieldTable; /** * Base resources. @@ -51,7 +75,11 @@ @Slf4j public class BaseResources { - protected String tenant = "public"; + @HeaderParam("tenant") + protected String tenant; + + @HeaderParam("x-vhost") + protected String xVhost; @Context protected ServletContext servletContext; @@ -67,6 +95,7 @@ public class BaseResources { private NamespaceService namespaceService; private NamespaceResources namespaceResources; + private TenantResources tenantResources; private ExchangeService exchangeService; @@ -76,6 +105,12 @@ public class BaseResources { private QueueContainer queueContainer; + private ManagedLedgerFactoryImpl managedLedgerFactory; + + private AmqpAdmin amqpAdmin; + private PulsarAdmin pulsarAdmin; + private PulsarClient pulsarClient; + protected AmqpProtocolHandler aop() { if (protocolHandler == null) { protocolHandler = (AmqpProtocolHandler) servletContext.getAttribute("aop"); @@ -83,6 +118,43 @@ protected AmqpProtocolHandler aop() { return protocolHandler; } + protected PulsarAdmin pulsarAdmin() { + if (pulsarAdmin == null) { + try { + pulsarAdmin = aop().getBrokerService().getPulsar().getAdminClient(); + } catch (PulsarServerException e) { + throw new AoPServiceRuntimeException(e); + } + } + return pulsarAdmin; + } + + protected PulsarClient pulsarClient() { + if (pulsarClient == null) { + try { + pulsarClient = aop().getBrokerService().getPulsar().getClient(); + } catch (PulsarServerException e) { + throw new AoPServiceRuntimeException(e); + } + } + return pulsarClient; + } + + protected AmqpAdmin amqpAdmin() { + if (amqpAdmin == null) { + amqpAdmin = aop().getAmqpBrokerService().getAmqpAdmin(); + } + return amqpAdmin; + } + + protected ManagedLedgerFactoryImpl managedLedgerFactory() { + if (managedLedgerFactory == null) { + managedLedgerFactory = (ManagedLedgerFactoryImpl) aop().getBrokerService().getManagedLedgerFactory(); + } + return managedLedgerFactory; + } + + protected NamespaceService namespaceService() { if (namespaceService == null) { namespaceService = aop().getBrokerService().getPulsar().getNamespaceService(); @@ -97,6 +169,13 @@ protected NamespaceResources namespaceResource() { return namespaceResources; } + protected TenantResources tenantResources() { + if (tenantResources == null) { + tenantResources = aop().getBrokerService().getPulsar().getPulsarResources().getTenantResources(); + } + return tenantResources; + } + protected ExchangeService exchangeService() { if (exchangeService == null) { exchangeService = aop().getAmqpBrokerService().getExchangeService(); @@ -126,11 +205,28 @@ protected QueueContainer queueContainer() { return queueContainer; } + protected NamespaceName getNamespaceName() { + return getNamespaceName(null); + } + + protected NamespaceName getNamespaceName(String vhost) { + vhost = StringUtils.isNotBlank(vhost) ? vhost : this.xVhost; + vhost = "/".equals(vhost) ? "default" : vhost; + NamespaceName namespaceName; + if (tenant == null) { + namespaceName = NamespaceName.get(vhost); + } else { + namespaceName = NamespaceName.get(tenant, vhost); + } + return namespaceName; + } + protected CompletableFuture> getVhostListAsync() { return namespaceResource().listNamespacesAsync(tenant) .thenApply(nsList -> { List vhostBeanList = new ArrayList<>(); nsList.forEach(ns -> { + ns = "default".equals(ns) ? "/" : ns; VhostBean bean = new VhostBean(); bean.setName(ns); vhostBeanList.add(bean); @@ -139,6 +235,54 @@ protected CompletableFuture> getVhostListAsync() { }); } + protected CompletableFuture> getTopicProperties(String namespaceName, String topicPrefix, + String topic) { + CompletableFuture> future = new CompletableFuture<>(); + // amqp/default/persistent/__amqp_exchange__direct_E2 + String path = namespaceName + "/persistent/" + topicPrefix + URLEncoder.encode(topic, StandardCharsets.UTF_8); + managedLedgerFactory().getMetaStore().getManagedLedgerInfo(path, false, + new MetaStore.MetaStoreCallback<>() { + @Override + public void operationComplete(MLDataFormats.ManagedLedgerInfo result, Stat stat) { + Map propertiesMap = new HashMap<>(); + if (result.getPropertiesCount() > 0) { + for (int i = 0; i < result.getPropertiesCount(); i++) { + MLDataFormats.KeyValue property = result.getProperties(i); + propertiesMap.put(property.getKey(), property.getValue()); + } + } + future.complete(propertiesMap); + } + + @Override + public void operationFailed(ManagedLedgerException.MetaStoreException e) { + log.error("Failed get TopicProperties.", e); + future.complete(null); + } + }); + return future; + } + + protected CompletableFuture> getAllVhostListAsync() { + return tenantResources().listTenantsAsync().thenCompose(tenantList -> { + Stream>> futureStream = tenantList.stream() + .map(s -> namespaceResource().listNamespacesAsync(s) + .thenApply(nsList -> { + List vhostBeanList = new ArrayList<>(); + nsList.forEach(ns -> { + VhostBean bean = new VhostBean(); + bean.setName(ns); + vhostBeanList.add(bean); + }); + return vhostBeanList; + })); + return FutureUtil.waitForAll(futureStream).thenApply(vhostBeans -> { + vhostBeans.sort(Comparator.comparing(VhostBean::getName)); + return vhostBeans; + }); + }); + } + private PulsarService pulsar() { return aop().getBrokerService().getPulsar(); } @@ -225,9 +369,70 @@ protected static void resumeAsyncResponseExceptionally(AsyncResponse asyncRespon asyncResponse.resume(new RestException(Response.Status.CONFLICT, "Concurrent modification")); } else if (realCause instanceof PulsarAdminException) { asyncResponse.resume(new RestException(((PulsarAdminException) realCause))); + } else if (realCause instanceof AoPServiceRuntimeException.NoSuchQueueException + || realCause instanceof AoPServiceRuntimeException.NoSuchExchangeException + || realCause instanceof AoPServiceRuntimeException.GetMessageException) { + asyncResponse.resume(new RestException(500, realCause.getMessage())); } else { asyncResponse.resume(new RestException(realCause)); } } + protected CompletableFuture queueBindAsync(NamespaceName namespaceName, String exchange, String queue, + BindingParams params) { + if (aop().getAmqpConfig().isAmqpMultiBundleEnable()) { + return exchangeService().queueBind(namespaceName, exchange, queue, params.getRoutingKey(), + params.getArguments()); + } else { + return queueService().queueBind(namespaceName, queue, exchange, params.getRoutingKey(), + false, FieldTable.convertToFieldTable(params.getArguments()), -1); + } + } + + protected CompletableFuture queueBindExchange(NamespaceName namespaceName, String exchange, String queue, + BindingParams params) { + if (aop().getAmqpConfig().isAmqpMultiBundleEnable()) { + return queueContainer().asyncGetQueue(namespaceName, queue, false) + .thenCompose(__ -> queueService().queueBind(namespaceName, queue, exchange, params)); + } else { + return queueService().queueBind(namespaceName, queue, exchange, params.getRoutingKey(), + false, FieldTable.convertToFieldTable(params.getArguments()), -1); + } + } + + protected CompletableFuture queueUnbindAsync(NamespaceName namespaceName, String exchange, String queue, + String propertiesKey) { + if (aop().getAmqpConfig().isAmqpMultiBundleEnable()) { + return exchangeService().queueUnBind(namespaceName, exchange, queue, propertiesKey, null); + } else { + return queueService().queueUnbind(namespaceName, queue, exchange, propertiesKey, + null, -1); + } + } + + protected CompletableFuture queueUnBindExchange(NamespaceName namespaceName, String exchange, String queue, + String propertiesKey) { + if (aop().getAmqpConfig().isAmqpMultiBundleEnable()) { + return queueService().queueUnBind(namespaceName, queue, exchange, propertiesKey); + } else { + return queueService().queueUnbind(namespaceName, queue, exchange, propertiesKey, + null, -1); + } + } + + public List getPageList(List list, int page, int pageSize) { + int p = pageSize == 0 ? 100 : pageSize; + return list.subList((page - 1) * p, Math.min(p * page, list.size())); + } + + public int getPageCount(int totalSize, int pageSize) { + if (totalSize == 0) { + return 0; + } + int page = pageSize == 0 ? 100 : pageSize; + return BigDecimal.valueOf(totalSize) + .divide(BigDecimal.valueOf(page), 0, RoundingMode.UP) + .intValue(); + } + } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/impl/ExchangeBase.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/impl/ExchangeBase.java index aac9af86..9a2f5233 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/impl/ExchangeBase.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/impl/ExchangeBase.java @@ -3,7 +3,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,8 +20,10 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.util.FutureUtil; @@ -29,7 +31,7 @@ /** * Exchange base. */ - +@Slf4j public class ExchangeBase extends BaseResources { protected CompletableFuture> getExchangeListAsync() { @@ -44,7 +46,7 @@ protected CompletableFuture> getExchangeListAsync() { }).thenApply(__ -> exchangeList); } - private CompletableFuture> getExchangeListAsync(String tenant, String ns) { + protected CompletableFuture> getExchangeListAsync(String tenant, String ns) { return namespaceService() .getFullListOfTopics(NamespaceName.get(tenant, ns)) .thenApply(list -> list.stream().filter(s -> @@ -79,15 +81,28 @@ protected CompletableFuture getExchangeBeanAsync(String vhost, Str }); } - protected CompletableFuture declareExchange(String vhost, String exchangeName, + protected CompletableFuture loadExchangeAsync(String vhost, String exchangeName) { + Map> exMap = + exchangeContainer().getExchangeMap().get(getNamespaceName(vhost)); + if (exMap != null) { + CompletableFuture future = exMap.get(exchangeName); + if (future != null) { + return future; + } + } + return exchangeContainer().asyncGetExchange(getNamespaceName(vhost), exchangeName, false, null); + } + + protected CompletableFuture declareExchange(NamespaceName namespaceName, String exchangeName, ExchangeDeclareParams declareParams) { - return exchangeService().exchangeDeclare(NamespaceName.get(tenant, vhost), exchangeName, + return exchangeService().exchangeDeclare(namespaceName, exchangeName, declareParams.getType(), declareParams.isPassive(), declareParams.isDurable(), declareParams.isAutoDelete(), declareParams.isInternal(), declareParams.getArguments()); } - protected CompletableFuture deleteExchange(String vhost, String exchangeName, boolean ifUnused) { - return exchangeService().exchangeDelete(NamespaceName.get(tenant, vhost), exchangeName, ifUnused); + protected CompletableFuture deleteExchange(NamespaceName namespaceName, String exchangeName, + boolean ifUnused) { + return exchangeService().exchangeDelete(namespaceName, exchangeName, ifUnused); } } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/impl/QueueBase.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/impl/QueueBase.java index 05d74dac..92aec877 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/impl/QueueBase.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/impl/QueueBase.java @@ -13,16 +13,25 @@ */ package io.streamnative.pulsar.handlers.amqp.admin.impl; +import static io.streamnative.pulsar.handlers.amqp.utils.ExchangeUtil.JSON_MAPPER; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.collect.Sets; import io.streamnative.pulsar.handlers.amqp.AmqpQueue; import io.streamnative.pulsar.handlers.amqp.admin.model.QueueBean; import io.streamnative.pulsar.handlers.amqp.admin.model.QueueDeclareParams; import io.streamnative.pulsar.handlers.amqp.admin.model.VhostBean; +import io.streamnative.pulsar.handlers.amqp.admin.model.rabbitmq.QueueBinds; +import io.streamnative.pulsar.handlers.amqp.impl.PersistentExchange; import io.streamnative.pulsar.handlers.amqp.impl.PersistentQueue; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.util.FutureUtil; @@ -30,6 +39,7 @@ /** * QueueBase. */ +@Slf4j public class QueueBase extends BaseResources { protected CompletableFuture> getQueueListAsync() { @@ -90,16 +100,67 @@ protected CompletableFuture getQueueBeanAsync(String vhost, String qu }); } - protected CompletableFuture declareQueueAsync(String vhost, String queue, - QueueDeclareParams declareParams) { - return queueService().queueDeclare(NamespaceName.get(tenant, vhost), queue, false, + protected CompletableFuture declareQueueAsync(NamespaceName namespaceName, String queue, + QueueDeclareParams declareParams) { + return queueService().queueDeclare(namespaceName, queue, declareParams.isPassive(), declareParams.isDurable(), declareParams.isExclusive(), declareParams.isAutoDelete(), true, declareParams.getArguments(), -1); } - protected CompletableFuture deleteQueueAsync(String vhost, String queue, boolean ifUnused, + protected CompletableFuture deleteQueueAsync(NamespaceName namespaceName, String queue, boolean ifUnused, boolean ifEmpty) { - return queueService().queueDelete(NamespaceName.get(tenant, vhost), queue, ifUnused, ifEmpty, -1); + return queueService().queueDelete(namespaceName, queue, ifUnused, ifEmpty, -1); } + protected CompletableFuture startExpirationDetection(String vhost, String queue) { + return queueContainer().asyncGetQueue(getNamespaceName(vhost), queue, false) + .thenAccept(amqpQueue -> { + if (amqpQueue instanceof PersistentQueue persistentQueue) { + persistentQueue.startMessageExpireChecker(); + } + }); + } + + protected CompletableFuture loadQueueAsync(String vhost, String queue) { + Map> queueMap = + queueContainer().getQueueMap().get(getNamespaceName(vhost)); + if (queueMap != null) { + CompletableFuture future = queueMap.get(queue); + if (future != null) { + return future; + } + } + return queueContainer().asyncGetQueue(getNamespaceName(vhost), queue, false); + } + + protected CompletableFuture> getQueueBindings(NamespaceName namespace, String queue) { + return getTopicProperties(namespace.toString(), PersistentQueue.TOPIC_PREFIX, queue).thenCompose(properties -> { + Set bindings = Sets.newHashSet(); + try { + if (properties.containsKey("BINDINGS")) { + List amqpQueueProperties = + JSON_MAPPER.readValue(properties.get("BINDINGS"), new TypeReference<>() { + }); + bindings.addAll(amqpQueueProperties); + } + } catch (JsonProcessingException e) { + return FutureUtil.failedFuture(e); + } + List binds = bindings.stream() + .map(binding -> { + QueueBinds queueBinds = new QueueBinds(); + queueBinds.setSource(binding.getSource()); + queueBinds.setDestination(binding.getDes()); + queueBinds.setRouting_key(binding.getKey()); + queueBinds.setProperties_key(binding.getKey()); + queueBinds.setVhost(namespace.getLocalName()); + queueBinds.setDestination_type(binding.getDesType()); + return queueBinds; + }).collect(Collectors.toList()); + return CompletableFuture.completedFuture(binds); + }).exceptionally(throwable -> { + log.error("Failed to save binding metadata for bind operation.", throwable); + return null; + }); + } } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/ExchangeDeclareParams.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/ExchangeDeclareParams.java index 9a10aba7..0bda362e 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/ExchangeDeclareParams.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/ExchangeDeclareParams.java @@ -32,5 +32,6 @@ public class ExchangeDeclareParams { private boolean internal; private boolean passive; private Map arguments; - + private String vhost; + private String name; } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/ExchangeDeleteParams.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/ExchangeDeleteParams.java new file mode 100644 index 00000000..86d64474 --- /dev/null +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/ExchangeDeleteParams.java @@ -0,0 +1,13 @@ +package io.streamnative.pulsar.handlers.amqp.admin.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class ExchangeDeleteParams { + + @JsonProperty("if-unused") + private boolean ifUnused; + private String name; + private String vhost; +} diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/PublishParams.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/PublishParams.java new file mode 100644 index 00000000..e4607ada --- /dev/null +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/PublishParams.java @@ -0,0 +1,38 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.streamnative.pulsar.handlers.amqp.admin.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Map; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +public class PublishParams { + + @JsonProperty("delivery_mode") + private String deliveryMode; + @JsonProperty("routing_key") + private String routingKey; + @JsonProperty("payload_encoding") + private String payloadEncoding; + private String name; + private String payload; + private String vhost; + private Map headers; + private Map properties; + private Map props; +} diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/QueueDeclareParams.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/QueueDeclareParams.java index a02be21a..1cef77f3 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/QueueDeclareParams.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/QueueDeclareParams.java @@ -29,7 +29,9 @@ public class QueueDeclareParams { private boolean autoDelete; private boolean durable; private boolean exclusive; + private boolean passive; private String node; private Map arguments; - + private String vhost; + private String name; } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/QueueDeleteParams.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/QueueDeleteParams.java new file mode 100644 index 00000000..149cf9cd --- /dev/null +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/QueueDeleteParams.java @@ -0,0 +1,29 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.streamnative.pulsar.handlers.amqp.admin.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class QueueDeleteParams { + + @JsonProperty("if-unused") + private boolean ifUnused; + @JsonProperty("if-empty") + private boolean ifEmpty; + private String mode; + private String name; + private String vhost; +} diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/QueueUnBindingParams.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/QueueUnBindingParams.java new file mode 100644 index 00000000..7742a544 --- /dev/null +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/QueueUnBindingParams.java @@ -0,0 +1,25 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.streamnative.pulsar.handlers.amqp.admin.model; + +import lombok.Data; + +@Data +public class QueueUnBindingParams { + private String destination; + private String destination_type; + private String properties_key; + private String source; + private String vhost; +} diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/ExchangeDetail.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/ExchangeDetail.java new file mode 100644 index 00000000..d81f5af4 --- /dev/null +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/ExchangeDetail.java @@ -0,0 +1,66 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.streamnative.pulsar.handlers.amqp.admin.model.rabbitmq; + +import com.google.common.collect.Lists; +import java.util.List; +import java.util.Map; +import lombok.Data; +import lombok.NoArgsConstructor; + +@NoArgsConstructor +@Data +public class ExchangeDetail { + + private Map arguments; + private boolean auto_delete; + private boolean durable; + private boolean internal; + private MessageStatsBean message_stats; + private String name; + private String fullName; + private String type; + private String user_who_performed_action; + private String vhost; + private List incoming; + private List outgoing; + + @NoArgsConstructor + @Data + public static class MessageStatsBean { + private long publish_in; + private PublishInDetailsBean publish_in_details; + private long publish_out; + private PublishOutDetailsBean publish_out_details; + + @NoArgsConstructor + @Data + public static class PublishInDetailsBean { + private double avg; + private double avg_rate; + private double rate; + private List samples = Lists.newArrayList(); + } + + @NoArgsConstructor + @Data + public static class PublishOutDetailsBean { + private double avg; + private double avg_rate; + private double rate; + private List samples = Lists.newArrayList(); + + } + } +} diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/ExchangeSource.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/ExchangeSource.java new file mode 100644 index 00000000..6c55773a --- /dev/null +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/ExchangeSource.java @@ -0,0 +1,32 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.streamnative.pulsar.handlers.amqp.admin.model.rabbitmq; + +import java.util.Map; +import lombok.Data; +import lombok.NoArgsConstructor; + +@NoArgsConstructor +@Data +public class ExchangeSource { + + private String source; + private String vhost; + private String destination; + private String destination_type; + private String routing_key; + private Map arguments; + private String properties_key; + +} diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/ExchangesList.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/ExchangesList.java new file mode 100644 index 00000000..ffa36516 --- /dev/null +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/ExchangesList.java @@ -0,0 +1,68 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.streamnative.pulsar.handlers.amqp.admin.model.rabbitmq; + +import java.util.List; +import java.util.Map; +import lombok.Data; +import lombok.NoArgsConstructor; + +@NoArgsConstructor +@Data +public class ExchangesList { + + private int filtered_count; + private int item_count; + private int page; + private int page_count; + private int page_size; + private int total_count; + private List items; + + @NoArgsConstructor + @Data + public static class ItemsBean { + private Map arguments; + private boolean auto_delete; + private boolean durable; + private boolean internal; + private String name; + private String fullName; + private String type; + private String user_who_performed_action; + private String vhost; + private MessageStatsBean message_stats; + + @NoArgsConstructor + @Data + public static class MessageStatsBean { + private long publish_in; + private PublishInDetailsBean publish_in_details; + private long publish_out; + private PublishOutDetailsBean publish_out_details; + + @NoArgsConstructor + @Data + public static class PublishInDetailsBean { + private double rate; + } + + @NoArgsConstructor + @Data + public static class PublishOutDetailsBean { + private double rate; + } + } + } +} diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/MessageStats.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/MessageStats.java new file mode 100644 index 00000000..b924c2c4 --- /dev/null +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/MessageStats.java @@ -0,0 +1,43 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.streamnative.pulsar.handlers.amqp.admin.model.rabbitmq; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +public class MessageStats { + + private long ack; + private String ack_details; + private long confirm; + private String confirm_details; + private long deliver; + private String deliver_details; + private long deliver_get; + private long deliver_get_details; + private long deliver_no_ack; + private long deliver_no_ack_details; + private long get; + private long get_details; + private long get_no_ack; + private long get_no_ack_details; + private long publish; + private long publish_details; + private long redeliver; + private long redeliver_details; + private long return_unroutable; + private long return_unroutable_details; +} diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/Nodes.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/Nodes.java new file mode 100644 index 00000000..4d49bfa6 --- /dev/null +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/Nodes.java @@ -0,0 +1,1761 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.streamnative.pulsar.handlers.amqp.admin.model.rabbitmq; + +import java.util.List; + +public class Nodes { + + private String os_pid; + private int fd_total; + private int sockets_total; + private long mem_limit; + private boolean mem_alarm; + private long disk_free_limit; + private boolean disk_free_alarm; + private int proc_total; + private String rates_mode; + private long uptime; + private int run_queue; + private int processors; + private String db_dir; + private int net_ticktime; + private String mem_calculation_strategy; + private String name; + private String type; + private boolean running; + private long mem_used; + private MemUsedDetailsBean mem_used_details; + private int fd_used; + private FdUsedDetailsBean fd_used_details; + private int sockets_used; + private SocketsUsedDetailsBean sockets_used_details; + private int proc_used; + private ProcUsedDetailsBean proc_used_details; + private long disk_free; + private DiskFreeDetailsBean disk_free_details; + private long gc_num; + private GcNumDetailsBean gc_num_details; + private long gc_bytes_reclaimed; + private GcBytesReclaimedDetailsBean gc_bytes_reclaimed_details; + private long context_switches; + private ContextSwitchesDetailsBean context_switches_details; + private int io_read_count; + private IoReadCountDetailsBean io_read_count_details; + private long io_read_bytes; + private IoReadBytesDetailsBean io_read_bytes_details; + private double io_read_avg_time; + private IoReadAvgTimeDetailsBean io_read_avg_time_details; + private int io_write_count; + private IoWriteCountDetailsBean io_write_count_details; + private long io_write_bytes; + private IoWriteBytesDetailsBean io_write_bytes_details; + private double io_write_avg_time; + private IoWriteAvgTimeDetailsBean io_write_avg_time_details; + private int io_sync_count; + private IoSyncCountDetailsBean io_sync_count_details; + private double io_sync_avg_time; + private IoSyncAvgTimeDetailsBean io_sync_avg_time_details; + private int io_seek_count; + private IoSeekCountDetailsBean io_seek_count_details; + private double io_seek_avg_time; + private IoSeekAvgTimeDetailsBean io_seek_avg_time_details; + private int io_reopen_count; + private IoReopenCountDetailsBean io_reopen_count_details; + private int mnesia_ram_tx_count; + private MnesiaRamTxCountDetailsBean mnesia_ram_tx_count_details; + private int mnesia_disk_tx_count; + private MnesiaDiskTxCountDetailsBean mnesia_disk_tx_count_details; + private int msg_store_read_count; + private MsgStoreReadCountDetailsBean msg_store_read_count_details; + private int msg_store_write_count; + private MsgStoreWriteCountDetailsBean msg_store_write_count_details; + private long queue_index_journal_write_count; + private QueueIndexJournalWriteCountDetailsBean queue_index_journal_write_count_details; + private int queue_index_write_count; + private QueueIndexWriteCountDetailsBean queue_index_write_count_details; + private int queue_index_read_count; + private QueueIndexReadCountDetailsBean queue_index_read_count_details; + private long io_file_handle_open_attempt_count; + private IoFileHandleOpenAttemptCountDetailsBean io_file_handle_open_attempt_count_details; + private double io_file_handle_open_attempt_avg_time; + private IoFileHandleOpenAttemptAvgTimeDetailsBean io_file_handle_open_attempt_avg_time_details; + private int connection_created; + private ConnectionCreatedDetailsBean connection_created_details; + private int connection_closed; + private ConnectionClosedDetailsBean connection_closed_details; + private int channel_created; + private ChannelCreatedDetailsBean channel_created_details; + private int channel_closed; + private ChannelClosedDetailsBean channel_closed_details; + private int queue_declared; + private QueueDeclaredDetailsBean queue_declared_details; + private int queue_created; + private QueueCreatedDetailsBean queue_created_details; + private int queue_deleted; + private QueueDeletedDetailsBean queue_deleted_details; + private MetricsGcQueueLengthBean metrics_gc_queue_length; + private List partitions; + private List exchange_types; + private List auth_mechanisms; + private List applications; + private List contexts; + private List log_files; + private List config_files; + private List enabled_plugins; + private List cluster_links; + + public String getOs_pid() { + return os_pid; + } + + public void setOs_pid(String os_pid) { + this.os_pid = os_pid; + } + + public int getFd_total() { + return fd_total; + } + + public void setFd_total(int fd_total) { + this.fd_total = fd_total; + } + + public int getSockets_total() { + return sockets_total; + } + + public void setSockets_total(int sockets_total) { + this.sockets_total = sockets_total; + } + + public long getMem_limit() { + return mem_limit; + } + + public void setMem_limit(long mem_limit) { + this.mem_limit = mem_limit; + } + + public boolean isMem_alarm() { + return mem_alarm; + } + + public void setMem_alarm(boolean mem_alarm) { + this.mem_alarm = mem_alarm; + } + + public long getDisk_free_limit() { + return disk_free_limit; + } + + public void setDisk_free_limit(long disk_free_limit) { + this.disk_free_limit = disk_free_limit; + } + + public boolean isDisk_free_alarm() { + return disk_free_alarm; + } + + public void setDisk_free_alarm(boolean disk_free_alarm) { + this.disk_free_alarm = disk_free_alarm; + } + + public int getProc_total() { + return proc_total; + } + + public void setProc_total(int proc_total) { + this.proc_total = proc_total; + } + + public String getRates_mode() { + return rates_mode; + } + + public void setRates_mode(String rates_mode) { + this.rates_mode = rates_mode; + } + + public long getUptime() { + return uptime; + } + + public void setUptime(long uptime) { + this.uptime = uptime; + } + + public int getRun_queue() { + return run_queue; + } + + public void setRun_queue(int run_queue) { + this.run_queue = run_queue; + } + + public int getProcessors() { + return processors; + } + + public void setProcessors(int processors) { + this.processors = processors; + } + + public String getDb_dir() { + return db_dir; + } + + public void setDb_dir(String db_dir) { + this.db_dir = db_dir; + } + + public int getNet_ticktime() { + return net_ticktime; + } + + public void setNet_ticktime(int net_ticktime) { + this.net_ticktime = net_ticktime; + } + + public String getMem_calculation_strategy() { + return mem_calculation_strategy; + } + + public void setMem_calculation_strategy(String mem_calculation_strategy) { + this.mem_calculation_strategy = mem_calculation_strategy; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public boolean isRunning() { + return running; + } + + public void setRunning(boolean running) { + this.running = running; + } + + public long getMem_used() { + return mem_used; + } + + public void setMem_used(long mem_used) { + this.mem_used = mem_used; + } + + public MemUsedDetailsBean getMem_used_details() { + return mem_used_details; + } + + public void setMem_used_details(MemUsedDetailsBean mem_used_details) { + this.mem_used_details = mem_used_details; + } + + public int getFd_used() { + return fd_used; + } + + public void setFd_used(int fd_used) { + this.fd_used = fd_used; + } + + public FdUsedDetailsBean getFd_used_details() { + return fd_used_details; + } + + public void setFd_used_details(FdUsedDetailsBean fd_used_details) { + this.fd_used_details = fd_used_details; + } + + public int getSockets_used() { + return sockets_used; + } + + public void setSockets_used(int sockets_used) { + this.sockets_used = sockets_used; + } + + public SocketsUsedDetailsBean getSockets_used_details() { + return sockets_used_details; + } + + public void setSockets_used_details(SocketsUsedDetailsBean sockets_used_details) { + this.sockets_used_details = sockets_used_details; + } + + public int getProc_used() { + return proc_used; + } + + public void setProc_used(int proc_used) { + this.proc_used = proc_used; + } + + public ProcUsedDetailsBean getProc_used_details() { + return proc_used_details; + } + + public void setProc_used_details(ProcUsedDetailsBean proc_used_details) { + this.proc_used_details = proc_used_details; + } + + public long getDisk_free() { + return disk_free; + } + + public void setDisk_free(long disk_free) { + this.disk_free = disk_free; + } + + public DiskFreeDetailsBean getDisk_free_details() { + return disk_free_details; + } + + public void setDisk_free_details(DiskFreeDetailsBean disk_free_details) { + this.disk_free_details = disk_free_details; + } + + public long getGc_num() { + return gc_num; + } + + public void setGc_num(long gc_num) { + this.gc_num = gc_num; + } + + public GcNumDetailsBean getGc_num_details() { + return gc_num_details; + } + + public void setGc_num_details(GcNumDetailsBean gc_num_details) { + this.gc_num_details = gc_num_details; + } + + public long getGc_bytes_reclaimed() { + return gc_bytes_reclaimed; + } + + public void setGc_bytes_reclaimed(long gc_bytes_reclaimed) { + this.gc_bytes_reclaimed = gc_bytes_reclaimed; + } + + public GcBytesReclaimedDetailsBean getGc_bytes_reclaimed_details() { + return gc_bytes_reclaimed_details; + } + + public void setGc_bytes_reclaimed_details(GcBytesReclaimedDetailsBean gc_bytes_reclaimed_details) { + this.gc_bytes_reclaimed_details = gc_bytes_reclaimed_details; + } + + public long getContext_switches() { + return context_switches; + } + + public void setContext_switches(long context_switches) { + this.context_switches = context_switches; + } + + public ContextSwitchesDetailsBean getContext_switches_details() { + return context_switches_details; + } + + public void setContext_switches_details(ContextSwitchesDetailsBean context_switches_details) { + this.context_switches_details = context_switches_details; + } + + public int getIo_read_count() { + return io_read_count; + } + + public void setIo_read_count(int io_read_count) { + this.io_read_count = io_read_count; + } + + public IoReadCountDetailsBean getIo_read_count_details() { + return io_read_count_details; + } + + public void setIo_read_count_details(IoReadCountDetailsBean io_read_count_details) { + this.io_read_count_details = io_read_count_details; + } + + public long getIo_read_bytes() { + return io_read_bytes; + } + + public void setIo_read_bytes(long io_read_bytes) { + this.io_read_bytes = io_read_bytes; + } + + public IoReadBytesDetailsBean getIo_read_bytes_details() { + return io_read_bytes_details; + } + + public void setIo_read_bytes_details(IoReadBytesDetailsBean io_read_bytes_details) { + this.io_read_bytes_details = io_read_bytes_details; + } + + public double getIo_read_avg_time() { + return io_read_avg_time; + } + + public void setIo_read_avg_time(double io_read_avg_time) { + this.io_read_avg_time = io_read_avg_time; + } + + public IoReadAvgTimeDetailsBean getIo_read_avg_time_details() { + return io_read_avg_time_details; + } + + public void setIo_read_avg_time_details(IoReadAvgTimeDetailsBean io_read_avg_time_details) { + this.io_read_avg_time_details = io_read_avg_time_details; + } + + public int getIo_write_count() { + return io_write_count; + } + + public void setIo_write_count(int io_write_count) { + this.io_write_count = io_write_count; + } + + public IoWriteCountDetailsBean getIo_write_count_details() { + return io_write_count_details; + } + + public void setIo_write_count_details(IoWriteCountDetailsBean io_write_count_details) { + this.io_write_count_details = io_write_count_details; + } + + public long getIo_write_bytes() { + return io_write_bytes; + } + + public void setIo_write_bytes(long io_write_bytes) { + this.io_write_bytes = io_write_bytes; + } + + public IoWriteBytesDetailsBean getIo_write_bytes_details() { + return io_write_bytes_details; + } + + public void setIo_write_bytes_details(IoWriteBytesDetailsBean io_write_bytes_details) { + this.io_write_bytes_details = io_write_bytes_details; + } + + public double getIo_write_avg_time() { + return io_write_avg_time; + } + + public void setIo_write_avg_time(double io_write_avg_time) { + this.io_write_avg_time = io_write_avg_time; + } + + public IoWriteAvgTimeDetailsBean getIo_write_avg_time_details() { + return io_write_avg_time_details; + } + + public void setIo_write_avg_time_details(IoWriteAvgTimeDetailsBean io_write_avg_time_details) { + this.io_write_avg_time_details = io_write_avg_time_details; + } + + public int getIo_sync_count() { + return io_sync_count; + } + + public void setIo_sync_count(int io_sync_count) { + this.io_sync_count = io_sync_count; + } + + public IoSyncCountDetailsBean getIo_sync_count_details() { + return io_sync_count_details; + } + + public void setIo_sync_count_details(IoSyncCountDetailsBean io_sync_count_details) { + this.io_sync_count_details = io_sync_count_details; + } + + public double getIo_sync_avg_time() { + return io_sync_avg_time; + } + + public void setIo_sync_avg_time(double io_sync_avg_time) { + this.io_sync_avg_time = io_sync_avg_time; + } + + public IoSyncAvgTimeDetailsBean getIo_sync_avg_time_details() { + return io_sync_avg_time_details; + } + + public void setIo_sync_avg_time_details(IoSyncAvgTimeDetailsBean io_sync_avg_time_details) { + this.io_sync_avg_time_details = io_sync_avg_time_details; + } + + public int getIo_seek_count() { + return io_seek_count; + } + + public void setIo_seek_count(int io_seek_count) { + this.io_seek_count = io_seek_count; + } + + public IoSeekCountDetailsBean getIo_seek_count_details() { + return io_seek_count_details; + } + + public void setIo_seek_count_details(IoSeekCountDetailsBean io_seek_count_details) { + this.io_seek_count_details = io_seek_count_details; + } + + public double getIo_seek_avg_time() { + return io_seek_avg_time; + } + + public void setIo_seek_avg_time(double io_seek_avg_time) { + this.io_seek_avg_time = io_seek_avg_time; + } + + public IoSeekAvgTimeDetailsBean getIo_seek_avg_time_details() { + return io_seek_avg_time_details; + } + + public void setIo_seek_avg_time_details(IoSeekAvgTimeDetailsBean io_seek_avg_time_details) { + this.io_seek_avg_time_details = io_seek_avg_time_details; + } + + public int getIo_reopen_count() { + return io_reopen_count; + } + + public void setIo_reopen_count(int io_reopen_count) { + this.io_reopen_count = io_reopen_count; + } + + public IoReopenCountDetailsBean getIo_reopen_count_details() { + return io_reopen_count_details; + } + + public void setIo_reopen_count_details(IoReopenCountDetailsBean io_reopen_count_details) { + this.io_reopen_count_details = io_reopen_count_details; + } + + public int getMnesia_ram_tx_count() { + return mnesia_ram_tx_count; + } + + public void setMnesia_ram_tx_count(int mnesia_ram_tx_count) { + this.mnesia_ram_tx_count = mnesia_ram_tx_count; + } + + public MnesiaRamTxCountDetailsBean getMnesia_ram_tx_count_details() { + return mnesia_ram_tx_count_details; + } + + public void setMnesia_ram_tx_count_details(MnesiaRamTxCountDetailsBean mnesia_ram_tx_count_details) { + this.mnesia_ram_tx_count_details = mnesia_ram_tx_count_details; + } + + public int getMnesia_disk_tx_count() { + return mnesia_disk_tx_count; + } + + public void setMnesia_disk_tx_count(int mnesia_disk_tx_count) { + this.mnesia_disk_tx_count = mnesia_disk_tx_count; + } + + public MnesiaDiskTxCountDetailsBean getMnesia_disk_tx_count_details() { + return mnesia_disk_tx_count_details; + } + + public void setMnesia_disk_tx_count_details(MnesiaDiskTxCountDetailsBean mnesia_disk_tx_count_details) { + this.mnesia_disk_tx_count_details = mnesia_disk_tx_count_details; + } + + public int getMsg_store_read_count() { + return msg_store_read_count; + } + + public void setMsg_store_read_count(int msg_store_read_count) { + this.msg_store_read_count = msg_store_read_count; + } + + public MsgStoreReadCountDetailsBean getMsg_store_read_count_details() { + return msg_store_read_count_details; + } + + public void setMsg_store_read_count_details(MsgStoreReadCountDetailsBean msg_store_read_count_details) { + this.msg_store_read_count_details = msg_store_read_count_details; + } + + public int getMsg_store_write_count() { + return msg_store_write_count; + } + + public void setMsg_store_write_count(int msg_store_write_count) { + this.msg_store_write_count = msg_store_write_count; + } + + public MsgStoreWriteCountDetailsBean getMsg_store_write_count_details() { + return msg_store_write_count_details; + } + + public void setMsg_store_write_count_details(MsgStoreWriteCountDetailsBean msg_store_write_count_details) { + this.msg_store_write_count_details = msg_store_write_count_details; + } + + public long getQueue_index_journal_write_count() { + return queue_index_journal_write_count; + } + + public void setQueue_index_journal_write_count(long queue_index_journal_write_count) { + this.queue_index_journal_write_count = queue_index_journal_write_count; + } + + public QueueIndexJournalWriteCountDetailsBean getQueue_index_journal_write_count_details() { + return queue_index_journal_write_count_details; + } + + public void setQueue_index_journal_write_count_details( + QueueIndexJournalWriteCountDetailsBean queue_index_journal_write_count_details) { + this.queue_index_journal_write_count_details = queue_index_journal_write_count_details; + } + + public int getQueue_index_write_count() { + return queue_index_write_count; + } + + public void setQueue_index_write_count(int queue_index_write_count) { + this.queue_index_write_count = queue_index_write_count; + } + + public QueueIndexWriteCountDetailsBean getQueue_index_write_count_details() { + return queue_index_write_count_details; + } + + public void setQueue_index_write_count_details(QueueIndexWriteCountDetailsBean queue_index_write_count_details) { + this.queue_index_write_count_details = queue_index_write_count_details; + } + + public int getQueue_index_read_count() { + return queue_index_read_count; + } + + public void setQueue_index_read_count(int queue_index_read_count) { + this.queue_index_read_count = queue_index_read_count; + } + + public QueueIndexReadCountDetailsBean getQueue_index_read_count_details() { + return queue_index_read_count_details; + } + + public void setQueue_index_read_count_details(QueueIndexReadCountDetailsBean queue_index_read_count_details) { + this.queue_index_read_count_details = queue_index_read_count_details; + } + + public long getIo_file_handle_open_attempt_count() { + return io_file_handle_open_attempt_count; + } + + public void setIo_file_handle_open_attempt_count(long io_file_handle_open_attempt_count) { + this.io_file_handle_open_attempt_count = io_file_handle_open_attempt_count; + } + + public IoFileHandleOpenAttemptCountDetailsBean getIo_file_handle_open_attempt_count_details() { + return io_file_handle_open_attempt_count_details; + } + + public void setIo_file_handle_open_attempt_count_details( + IoFileHandleOpenAttemptCountDetailsBean io_file_handle_open_attempt_count_details) { + this.io_file_handle_open_attempt_count_details = io_file_handle_open_attempt_count_details; + } + + public double getIo_file_handle_open_attempt_avg_time() { + return io_file_handle_open_attempt_avg_time; + } + + public void setIo_file_handle_open_attempt_avg_time(double io_file_handle_open_attempt_avg_time) { + this.io_file_handle_open_attempt_avg_time = io_file_handle_open_attempt_avg_time; + } + + public IoFileHandleOpenAttemptAvgTimeDetailsBean getIo_file_handle_open_attempt_avg_time_details() { + return io_file_handle_open_attempt_avg_time_details; + } + + public void setIo_file_handle_open_attempt_avg_time_details( + IoFileHandleOpenAttemptAvgTimeDetailsBean io_file_handle_open_attempt_avg_time_details) { + this.io_file_handle_open_attempt_avg_time_details = io_file_handle_open_attempt_avg_time_details; + } + + public int getConnection_created() { + return connection_created; + } + + public void setConnection_created(int connection_created) { + this.connection_created = connection_created; + } + + public ConnectionCreatedDetailsBean getConnection_created_details() { + return connection_created_details; + } + + public void setConnection_created_details(ConnectionCreatedDetailsBean connection_created_details) { + this.connection_created_details = connection_created_details; + } + + public int getConnection_closed() { + return connection_closed; + } + + public void setConnection_closed(int connection_closed) { + this.connection_closed = connection_closed; + } + + public ConnectionClosedDetailsBean getConnection_closed_details() { + return connection_closed_details; + } + + public void setConnection_closed_details(ConnectionClosedDetailsBean connection_closed_details) { + this.connection_closed_details = connection_closed_details; + } + + public int getChannel_created() { + return channel_created; + } + + public void setChannel_created(int channel_created) { + this.channel_created = channel_created; + } + + public ChannelCreatedDetailsBean getChannel_created_details() { + return channel_created_details; + } + + public void setChannel_created_details(ChannelCreatedDetailsBean channel_created_details) { + this.channel_created_details = channel_created_details; + } + + public int getChannel_closed() { + return channel_closed; + } + + public void setChannel_closed(int channel_closed) { + this.channel_closed = channel_closed; + } + + public ChannelClosedDetailsBean getChannel_closed_details() { + return channel_closed_details; + } + + public void setChannel_closed_details(ChannelClosedDetailsBean channel_closed_details) { + this.channel_closed_details = channel_closed_details; + } + + public int getQueue_declared() { + return queue_declared; + } + + public void setQueue_declared(int queue_declared) { + this.queue_declared = queue_declared; + } + + public QueueDeclaredDetailsBean getQueue_declared_details() { + return queue_declared_details; + } + + public void setQueue_declared_details(QueueDeclaredDetailsBean queue_declared_details) { + this.queue_declared_details = queue_declared_details; + } + + public int getQueue_created() { + return queue_created; + } + + public void setQueue_created(int queue_created) { + this.queue_created = queue_created; + } + + public QueueCreatedDetailsBean getQueue_created_details() { + return queue_created_details; + } + + public void setQueue_created_details(QueueCreatedDetailsBean queue_created_details) { + this.queue_created_details = queue_created_details; + } + + public int getQueue_deleted() { + return queue_deleted; + } + + public void setQueue_deleted(int queue_deleted) { + this.queue_deleted = queue_deleted; + } + + public QueueDeletedDetailsBean getQueue_deleted_details() { + return queue_deleted_details; + } + + public void setQueue_deleted_details(QueueDeletedDetailsBean queue_deleted_details) { + this.queue_deleted_details = queue_deleted_details; + } + + public MetricsGcQueueLengthBean getMetrics_gc_queue_length() { + return metrics_gc_queue_length; + } + + public void setMetrics_gc_queue_length(MetricsGcQueueLengthBean metrics_gc_queue_length) { + this.metrics_gc_queue_length = metrics_gc_queue_length; + } + + public List getPartitions() { + return partitions; + } + + public void setPartitions(List partitions) { + this.partitions = partitions; + } + + public List getExchange_types() { + return exchange_types; + } + + public void setExchange_types(List exchange_types) { + this.exchange_types = exchange_types; + } + + public List getAuth_mechanisms() { + return auth_mechanisms; + } + + public void setAuth_mechanisms(List auth_mechanisms) { + this.auth_mechanisms = auth_mechanisms; + } + + public List getApplications() { + return applications; + } + + public void setApplications(List applications) { + this.applications = applications; + } + + public List getContexts() { + return contexts; + } + + public void setContexts(List contexts) { + this.contexts = contexts; + } + + public List getLog_files() { + return log_files; + } + + public void setLog_files(List log_files) { + this.log_files = log_files; + } + + public List getConfig_files() { + return config_files; + } + + public void setConfig_files(List config_files) { + this.config_files = config_files; + } + + public List getEnabled_plugins() { + return enabled_plugins; + } + + public void setEnabled_plugins(List enabled_plugins) { + this.enabled_plugins = enabled_plugins; + } + + public List getCluster_links() { + return cluster_links; + } + + public void setCluster_links(List cluster_links) { + this.cluster_links = cluster_links; + } + + public static class MemUsedDetailsBean { + /** + * rate : -2542796.8 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class FdUsedDetailsBean { + /** + * rate : 0.0 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class SocketsUsedDetailsBean { + /** + * rate : 0.0 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class ProcUsedDetailsBean { + /** + * rate : 0.0 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class DiskFreeDetailsBean { + /** + * rate : 0.0 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class GcNumDetailsBean { + /** + * rate : 509.6 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class GcBytesReclaimedDetailsBean { + /** + * rate : 2.34351632E7 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class ContextSwitchesDetailsBean { + /** + * rate : 8296.8 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class IoReadCountDetailsBean { + /** + * rate : 0.0 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class IoReadBytesDetailsBean { + /** + * rate : 0.0 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class IoReadAvgTimeDetailsBean { + /** + * rate : 0.0 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class IoWriteCountDetailsBean { + /** + * rate : 0.6 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class IoWriteBytesDetailsBean { + /** + * rate : 612.8 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class IoWriteAvgTimeDetailsBean { + /** + * rate : 0.08966666666666667 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class IoSyncCountDetailsBean { + /** + * rate : 0.6 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class IoSyncAvgTimeDetailsBean { + /** + * rate : 1.7743333333333333 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class IoSeekCountDetailsBean { + /** + * rate : 0.6 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class IoSeekAvgTimeDetailsBean { + /** + * rate : 0.019 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class IoReopenCountDetailsBean { + /** + * rate : 0.0 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class MnesiaRamTxCountDetailsBean { + /** + * rate : 0.0 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class MnesiaDiskTxCountDetailsBean { + /** + * rate : 0.0 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class MsgStoreReadCountDetailsBean { + /** + * rate : 0.0 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class MsgStoreWriteCountDetailsBean { + /** + * rate : 0.0 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class QueueIndexJournalWriteCountDetailsBean { + /** + * rate : 1.2 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class QueueIndexWriteCountDetailsBean { + /** + * rate : 0.4 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class QueueIndexReadCountDetailsBean { + /** + * rate : 0.0 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class IoFileHandleOpenAttemptCountDetailsBean { + /** + * rate : 2.6 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class IoFileHandleOpenAttemptAvgTimeDetailsBean { + /** + * rate : 0.015769230769230768 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class ConnectionCreatedDetailsBean { + /** + * rate : 0.0 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class ConnectionClosedDetailsBean { + /** + * rate : 1.6 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class ChannelCreatedDetailsBean { + /** + * rate : 0.0 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class ChannelClosedDetailsBean { + /** + * rate : 0.0 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class QueueDeclaredDetailsBean { + /** + * rate : 0.0 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class QueueCreatedDetailsBean { + /** + * rate : 0.0 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class QueueDeletedDetailsBean { + /** + * rate : 0.0 + */ + + private double rate; + + public double getRate() { + return rate; + } + + public void setRate(double rate) { + this.rate = rate; + } + } + + public static class MetricsGcQueueLengthBean { + /** + * connection_closed : 0 + * channel_closed : 0 + * consumer_deleted : 0 + * exchange_deleted : 0 + * queue_deleted : 0 + * vhost_deleted : 0 + * node_node_deleted : 0 + * channel_consumer_deleted : 0 + */ + + private int connection_closed; + private int channel_closed; + private int consumer_deleted; + private int exchange_deleted; + private int queue_deleted; + private int vhost_deleted; + private int node_node_deleted; + private int channel_consumer_deleted; + + public int getConnection_closed() { + return connection_closed; + } + + public void setConnection_closed(int connection_closed) { + this.connection_closed = connection_closed; + } + + public int getChannel_closed() { + return channel_closed; + } + + public void setChannel_closed(int channel_closed) { + this.channel_closed = channel_closed; + } + + public int getConsumer_deleted() { + return consumer_deleted; + } + + public void setConsumer_deleted(int consumer_deleted) { + this.consumer_deleted = consumer_deleted; + } + + public int getExchange_deleted() { + return exchange_deleted; + } + + public void setExchange_deleted(int exchange_deleted) { + this.exchange_deleted = exchange_deleted; + } + + public int getQueue_deleted() { + return queue_deleted; + } + + public void setQueue_deleted(int queue_deleted) { + this.queue_deleted = queue_deleted; + } + + public int getVhost_deleted() { + return vhost_deleted; + } + + public void setVhost_deleted(int vhost_deleted) { + this.vhost_deleted = vhost_deleted; + } + + public int getNode_node_deleted() { + return node_node_deleted; + } + + public void setNode_node_deleted(int node_node_deleted) { + this.node_node_deleted = node_node_deleted; + } + + public int getChannel_consumer_deleted() { + return channel_consumer_deleted; + } + + public void setChannel_consumer_deleted(int channel_consumer_deleted) { + this.channel_consumer_deleted = channel_consumer_deleted; + } + } + + public static class ExchangeTypesBean { + /** + * name : direct + * description : AMQP direct exchange, as per the AMQP specification + * enabled : true + */ + + private String name; + private String description; + private boolean enabled; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + } + + public static class AuthMechanismsBean { + /** + * name : RABBIT-CR-DEMO + * description : RabbitMQ Demo challenge-response authentication mechanism + * enabled : false + */ + + private String name; + private String description; + private boolean enabled; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + } + + public static class ApplicationsBean { + /** + * name : accept + * description : Accept header(s) for Erlang/Elixir + * version : 0.3.3 + */ + + private String name; + private String description; + private String version; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + } + + public static class ContextsBean { + /** + * description : RabbitMQ Management + * path : / + * cowboy_opts : [{sendfile,false}] + * ip : 10.70.20.123 + * port : 15672 + * ssl_opts : {"keyfile":"/opt/dms/maintain/cert/dms.key","certfile":"/opt/dms/maintain/cert/dms.crt","cacertfile":"/opt/dms/maintain/cert/ca.crt"} + */ + + private String description; + private String path; + private String cowboy_opts; + private String ip; + private String port; + private SslOptsBean ssl_opts; + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getCowboy_opts() { + return cowboy_opts; + } + + public void setCowboy_opts(String cowboy_opts) { + this.cowboy_opts = cowboy_opts; + } + + public String getIp() { + return ip; + } + + public void setIp(String ip) { + this.ip = ip; + } + + public String getPort() { + return port; + } + + public void setPort(String port) { + this.port = port; + } + + public SslOptsBean getSsl_opts() { + return ssl_opts; + } + + public void setSsl_opts(SslOptsBean ssl_opts) { + this.ssl_opts = ssl_opts; + } + + public static class SslOptsBean { + /** + * keyfile : /opt/dms/maintain/cert/dms.key + * certfile : /opt/dms/maintain/cert/dms.crt + * cacertfile : /opt/dms/maintain/cert/ca.crt + */ + + private String keyfile; + private String certfile; + private String cacertfile; + + public String getKeyfile() { + return keyfile; + } + + public void setKeyfile(String keyfile) { + this.keyfile = keyfile; + } + + public String getCertfile() { + return certfile; + } + + public void setCertfile(String certfile) { + this.certfile = certfile; + } + + public String getCacertfile() { + return cacertfile; + } + + public void setCacertfile(String cacertfile) { + this.cacertfile = cacertfile; + } + } + } +} diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/OverviewBean.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/OverviewBean.java new file mode 100644 index 00000000..8c5701da --- /dev/null +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/OverviewBean.java @@ -0,0 +1,177 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.streamnative.pulsar.handlers.amqp.admin.model.rabbitmq; + +import com.google.common.collect.Lists; +import java.util.List; +import lombok.Data; +import lombok.NoArgsConstructor; + + +@NoArgsConstructor +@Data +public class OverviewBean { + + private String management_version; + private String rates_mode; + private SampleRetentionPoliciesBean sample_retention_policies; + private String rabbitmq_version; + private String cluster_name; + private String erlang_version; + private String erlang_full_version; + private MessageStatsBean message_stats; + private ChurnRatesBean churn_rates; + private QueueTotalsBean queue_totals; + private ObjectTotalsBean object_totals; + private int statistics_db_event_queue; + private String node; + private List exchange_types; + private List listeners; + private List contexts; + + @NoArgsConstructor + @Data + public static class SampleRetentionPoliciesBean { + private List global; + private List basic; + private List detailed; + } + + @NoArgsConstructor + @Data + public static class RateBean { + private double rate; + private double avg_rate; + private double avg; + private List samples = Lists.newArrayList(); + } + + @NoArgsConstructor + @Data + public static class MessageStatsBean { + private long ack; + private RateBean ack_details; + private long confirm; + private RateBean confirm_details; + private long deliver; + private RateBean deliver_details; + private long deliver_get; + private RateBean deliver_get_details; + private int deliver_no_ack; + private RateBean deliver_no_ack_details; + private int disk_reads; + private RateBean disk_reads_details; + private long disk_writes; + private RateBean disk_writes_details; + private int get; + private RateBean get_details; + private int get_no_ack; + private RateBean get_no_ack_details; + private long publish; + private RateBean publish_details; + private long redeliver; + private RateBean redeliver_details; + private int return_unroutable; + private RateBean return_unroutable_details; + } + + @NoArgsConstructor + @Data + public static class ChurnRatesBean { + private int channel_closed; + private RateBean channel_closed_details; + private int channel_created; + private RateBean channel_created_details; + private int connection_closed; + private RateBean connection_closed_details; + private int connection_created; + private RateBean connection_created_details; + private int queue_created; + private RateBean queue_created_details; + private int queue_declared; + private RateBean queue_declared_details; + private int queue_deleted; + private RateBean queue_deleted_details; + + } + + @NoArgsConstructor + @Data + public static class QueueTotalsBean { + private long messages; + private RateBean messages_details; + private long messages_ready; + private RateBean messages_ready_details; + private long messages_unacknowledged; + private RateBean messages_unacknowledged_details; + + } + + @NoArgsConstructor + @Data + public static class ObjectTotalsBean { + private int channels; + private int connections; + private int consumers; + private int exchanges; + private int queues; + } + + @NoArgsConstructor + @Data + public static class ExchangeTypesBean { + private String name; + private String description; + private boolean enabled; + } + + @NoArgsConstructor + @Data + public static class ListenersBean { + private String node; + private String protocol; + private String ip_address; + private int port; + private SocketOptsBean socket_opts; + + @NoArgsConstructor + @Data + public static class SocketOptsBean { + private int backlog; + private boolean nodelay; + private boolean exit_on_close; + private List linger; + } + } + + @NoArgsConstructor + @Data + public static class ContextsBean { + private SslOptsBean ssl_opts; + private String node; + private String description; + private String path; + private String cowboy_opts; + private String ip; + private String port; + + @NoArgsConstructor + @Data + public static class SslOptsBean { + private String keyfile; + private String certfile; + private String cacertfile; + } + } +} diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/QueueBinds.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/QueueBinds.java new file mode 100644 index 00000000..e0bdd92a --- /dev/null +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/QueueBinds.java @@ -0,0 +1,31 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.streamnative.pulsar.handlers.amqp.admin.model.rabbitmq; + +import java.util.Map; +import lombok.Data; +import lombok.NoArgsConstructor; + +@NoArgsConstructor +@Data +public class QueueBinds { + + private String source; + private String vhost; + private String destination; + private String destination_type; + private String routing_key; + private Map arguments; + private String properties_key; +} diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/QueueDetail.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/QueueDetail.java new file mode 100644 index 00000000..ea534930 --- /dev/null +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/QueueDetail.java @@ -0,0 +1,149 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.streamnative.pulsar.handlers.amqp.admin.model.rabbitmq; + +import com.google.common.collect.Lists; +import java.util.List; +import java.util.Map; +import lombok.Data; +import lombok.NoArgsConstructor; + +@NoArgsConstructor +@Data +public class QueueDetail { + + private Map arguments; + private boolean auto_delete; + private Map backing_queue_status; + private Object consumer_utilisation; + private int consumers; + private boolean durable; + private Map effective_policy_definition; + private boolean exclusive; + private Object exclusive_consumer_tag; + private GarbageCollectionBean garbage_collection; + private Object head_message_timestamp; + private String idle_since; + private int memory; + private int message_bytes; + private int message_bytes_paged_out; + private int message_bytes_persistent; + private int message_bytes_ram; + private int message_bytes_ready; + private int message_bytes_unacknowledged; + private MessageStatsBean message_stats; + private long messages; + private RateBean messages_details; + private int messages_paged_out; + private int messages_persistent; + private int messages_ram; + private long messages_ready; + private RateBean messages_ready_details; + private int messages_ready_ram; + private long messages_unacknowledged; + private RateBean messages_unacknowledged_details; + private int messages_unacknowledged_ram; + private String name; + private String fullName; + private String node; + private Object operator_policy; + private Object policy; + private Object recoverable_slaves; + private long reductions; + private RateBean reductions_details; + private String state; + private String vhost; + private List consumer_details; + private List deliveries; + private List incoming; + + @NoArgsConstructor + @Data + public static class GarbageCollectionBean { + private int fullsweep_after; + private int max_heap_size; + private int min_bin_vheap_size; + private int min_heap_size; + private int minor_gcs; + } + @NoArgsConstructor + @Data + public static class MessageStatsBean { + private int ack; + private RateBean ack_details; + private long deliver; + private RateBean deliver_details; + private int deliver_get; + private RateBean deliver_get_details; + private int deliver_no_ack; + private RateBean deliver_no_ack_details; + private int get; + private RateBean get_details; + private int get_no_ack; + private RateBean get_no_ack_details; + private long publish; + private RateBean publish_details; + private int redeliver; + private RateBean redeliver_details; + + + } + @NoArgsConstructor + @Data + public static class RateBean { + private double avg; + private double avg_rate; + private double rate; + private List samples = Lists.newArrayList(); + + public void setSamples(List samples) { + if (samples == null || samples.isEmpty()) { + return; + } + this.samples = samples; + } + } + + @NoArgsConstructor + @Data + public static class ConsumerDetailsBean { + private Map arguments; + private ChannelDetailsBean channel_details; + private boolean ack_required; + private String consumer_tag; + private boolean exclusive; + private int prefetch_count; + private QueueBean queue; + private String activity_status; + + @NoArgsConstructor + @Data + public static class ChannelDetailsBean { + private String connection_name; + private String name; + private String node; + private int number; + private String peer_host; + private int peer_port; + private String user; + } + + @NoArgsConstructor + @Data + public static class QueueBean { + private String name; + private String vhost; + } + } +} diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/QueuesList.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/QueuesList.java new file mode 100644 index 00000000..0d46c54a --- /dev/null +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/QueuesList.java @@ -0,0 +1,115 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.streamnative.pulsar.handlers.amqp.admin.model.rabbitmq; + +import java.util.List; +import java.util.Map; +import lombok.Data; +import lombok.NoArgsConstructor; + +@NoArgsConstructor +@Data +public class QueuesList { + + + private int filtered_count; + private int item_count; + private int page; + private int page_count; + private int page_size; + private int total_count; + private List items; + + @NoArgsConstructor + @Data + public static class ItemsBean { + private String fullName; + private Map arguments; + private boolean auto_delete; + private Map backing_queue_status; + private Object consumer_utilisation; + private int consumers; + private boolean durable; + private Map effective_policy_definition; + private boolean exclusive; + private Object exclusive_consumer_tag; + private GarbageCollectionBean garbage_collection; + private Object head_message_timestamp; + private String idle_since; + private int memory; + private int message_bytes; + private int message_bytes_paged_out; + private int message_bytes_persistent; + private int message_bytes_ram; + private int message_bytes_ready; + private int message_bytes_unacknowledged; + private long messages; + private RateBean messages_details; + private int messages_paged_out; + private int messages_persistent; + private int messages_ram; + private long messages_ready; + private RateBean messages_ready_details; + private int messages_ready_ram; + private long messages_unacknowledged; + private RateBean messages_unacknowledged_details; + private int messages_unacknowledged_ram; + private String name; + private String node; + private Object operator_policy; + private Object policy; + private Object recoverable_slaves; + private long reductions; + private RateBean reductions_details; + private String state; + private String vhost; + private MessageStatsBean message_stats; + + @NoArgsConstructor + @Data + public static class GarbageCollectionBean { + private int fullsweep_after; + private int max_heap_size; + private int min_bin_vheap_size; + private int min_heap_size; + private int minor_gcs; + } + + @NoArgsConstructor + @Data + public static class MessageStatsBean { + private int ack; + private RateBean ack_details; + private int deliver; + private RateBean deliver_details; + private int deliver_get; + private RateBean deliver_get_details; + private int deliver_no_ack; + private RateBean deliver_no_ack_details; + private int get; + private RateBean get_details; + private int get_no_ack; + private RateBean get_no_ack_details; + private int redeliver; + private RateBean redeliver_details; + private long publish; + private RateBean publish_details; + } + } + @NoArgsConstructor + @Data + public static class RateBean { + private double rate; + } +} diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/SamplesBean.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/SamplesBean.java new file mode 100644 index 00000000..9402e0ec --- /dev/null +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/SamplesBean.java @@ -0,0 +1,26 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.streamnative.pulsar.handlers.amqp.admin.model.rabbitmq; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@NoArgsConstructor +@AllArgsConstructor +@Data +public class SamplesBean { + private double sample; + private double timestamp; +} diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/Whomi.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/Whomi.java new file mode 100644 index 00000000..9295a9aa --- /dev/null +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/admin/model/rabbitmq/Whomi.java @@ -0,0 +1,40 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.streamnative.pulsar.handlers.amqp.admin.model.rabbitmq; + +import com.fasterxml.jackson.core.JsonProcessingException; +import io.streamnative.pulsar.handlers.amqp.utils.JsonUtil; +import java.util.HashMap; +import java.util.Map; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Path("/whoami") +@Produces(MediaType.APPLICATION_JSON) +public class Whomi { + + @GET + public Response whomi() throws JsonProcessingException { + Map map = new HashMap<>(); + map.put("name", "root"); + map.put("tags", "administrator"); + String str = JsonUtil.toString(map); + return Response.ok(str).build(); + } +} diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/common/exception/AoPServiceRuntimeException.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/common/exception/AoPServiceRuntimeException.java index 86bc143f..bdfee7df 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/common/exception/AoPServiceRuntimeException.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/common/exception/AoPServiceRuntimeException.java @@ -28,6 +28,11 @@ public ProducerCreationRuntimeException(Exception e) { super(e); } } + public static class ReaderCreationRuntimeException extends AoPServiceRuntimeException { + public ReaderCreationRuntimeException(Exception e) { + super(e); + } + } public static class ExchangeParameterException extends AoPServiceRuntimeException{ public ExchangeParameterException(String message) { @@ -47,4 +52,31 @@ public MessageRouteException(String message) { } } + public static class NotSupportedExchangeTypeException extends AoPServiceRuntimeException{ + public NotSupportedExchangeTypeException(String message) { + super(message); + } + } + + public static class NoSuchQueueException extends AoPServiceRuntimeException{ + public NoSuchQueueException(String message) { + super(message); + } + } + + public static class NoSuchExchangeException extends AoPServiceRuntimeException{ + public NoSuchExchangeException(String message) { + super(message); + } + } + + public static class GetMessageException extends AoPServiceRuntimeException{ + public GetMessageException(String message) { + super(message); + } + + public GetMessageException(Exception e) { + super(e); + } + } } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/impl/InMemoryExchange.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/impl/InMemoryExchange.java index 9764c2ea..9ffcd035 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/impl/InMemoryExchange.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/impl/InMemoryExchange.java @@ -45,13 +45,13 @@ public class InMemoryExchange extends AbstractAmqpExchange { private long currentEntryId; public InMemoryExchange(String exchangeName, AmqpExchange.Type exchangeType, boolean autoDelete) { - super(exchangeName, exchangeType, new HashSet<>(), false, autoDelete, false, null); + super(exchangeName, exchangeType, new HashSet<>(), false, autoDelete, false, null, null); this.currentLedgerId = 1L; } public InMemoryExchange(String exchangeName, AmqpExchange.Type exchangeType, boolean autoDelete, Map arguments) { - super(exchangeName, exchangeType, new HashSet<>(), false, autoDelete, false, arguments); + super(exchangeName, exchangeType, new HashSet<>(), false, autoDelete, false, arguments, null); this.currentLedgerId = 1L; } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/impl/InMemoryQueue.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/impl/InMemoryQueue.java index b8f536be..3a7de256 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/impl/InMemoryQueue.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/impl/InMemoryQueue.java @@ -34,8 +34,8 @@ public InMemoryQueue(String queueName, long connectionId) { super(queueName, false, connectionId); } - public InMemoryQueue(String queueName, long connectionId, boolean exclusive, boolean autoDelete) { - super(queueName, false, connectionId, exclusive, autoDelete); + public InMemoryQueue(String queueName, long connectionId, boolean exclusive, boolean autoDelete, Map properties) { + super(queueName, false, connectionId, exclusive, autoDelete, properties); } @Override diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/impl/PersistentExchange.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/impl/PersistentExchange.java index 3ffcf40a..abc5adfc 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/impl/PersistentExchange.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/impl/PersistentExchange.java @@ -3,7 +3,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -25,6 +25,7 @@ import io.streamnative.pulsar.handlers.amqp.AmqpExchangeReplicator; import io.streamnative.pulsar.handlers.amqp.AmqpQueue; import io.streamnative.pulsar.handlers.amqp.ExchangeMessageRouter; +import io.streamnative.pulsar.handlers.amqp.admin.AmqpAdmin; import io.streamnative.pulsar.handlers.amqp.utils.MessageConvertUtils; import io.streamnative.pulsar.handlers.amqp.utils.PulsarTopicMetadataUtils; import java.io.Serializable; @@ -39,6 +40,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.EqualsAndHashCode; +import lombok.Getter; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.AsyncCallbacks; @@ -51,6 +53,7 @@ import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.common.api.proto.KeyValue; @@ -74,52 +77,70 @@ public class PersistentExchange extends AbstractAmqpExchange { public static final String ARGUMENTS = "ARGUMENTS"; public static final String TOPIC_PREFIX = "__amqp_exchange__"; private static final String BINDINGS = "BINDINGS"; + public static final String X_DELAYED_TYPE = "x-delayed-type"; + @Getter + private final boolean existDelayedType; private PersistentTopic persistentTopic; - private final ConcurrentOpenHashMap> cursors; + private ConcurrentOpenHashMap> cursors; private AmqpExchangeReplicator messageReplicator; private AmqpEntryWriter amqpEntryWriter; private ExchangeMessageRouter exchangeMessageRouter; + @Getter private Set bindings; + @Getter + private PulsarClient pulsarClient; + @NoArgsConstructor @AllArgsConstructor - @EqualsAndHashCode + @EqualsAndHashCode(exclude = {"source", "arguments"}) @Data - static class Binding implements Serializable { + public static class Binding implements Serializable { private String des; private String desType; private String key; + private String source; private Map arguments; } - public PersistentExchange(String exchangeName, Type type, PersistentTopic persistentTopic, + public PersistentExchange(String exchangeName, Map properties, Type type, + PersistentTopic persistentTopic, boolean durable, boolean autoDelete, boolean internal, Map arguments, - ExecutorService routeExecutor, int routeQueueSize, boolean amqpMultiBundleEnable) + ExecutorService routeExecutor, int routeQueueSize, boolean amqpMultiBundleEnable, + AmqpAdmin amqpAdmin, PulsarClient pulsarClient) throws JsonProcessingException { - super(exchangeName, type, Sets.newConcurrentHashSet(), durable, autoDelete, internal, arguments); + super(exchangeName, type, Sets.newConcurrentHashSet(), durable, autoDelete, internal, arguments, properties); + this.pulsarClient = pulsarClient; this.persistentTopic = persistentTopic; + this.existDelayedType = arguments != null && arguments.containsKey(X_DELAYED_TYPE); topicNameValidate(); - cursors = new ConcurrentOpenHashMap<>(16, 1); - for (ManagedCursor cursor : persistentTopic.getManagedLedger().getCursors()) { - cursors.put(cursor.getName(), CompletableFuture.completedFuture(cursor)); - log.info("PersistentExchange {} recover cursor {}", persistentTopic.getName(), cursor.toString()); - cursor.setInactive(); - } if (amqpMultiBundleEnable) { bindings = Sets.newConcurrentHashSet(); if (persistentTopic.getManagedLedger().getProperties().containsKey(BINDINGS)) { List amqpQueueProperties = JSON_MAPPER.readValue( - persistentTopic.getManagedLedger().getProperties().get(BINDINGS), new TypeReference<>() {}); + persistentTopic.getManagedLedger().getProperties().get(BINDINGS), new TypeReference<>() { + }); this.bindings.addAll(amqpQueueProperties); } this.exchangeMessageRouter = ExchangeMessageRouter.getInstance(this, routeExecutor); + List> futures = new ArrayList<>(); + NamespaceName namespaceName = TopicName.get(persistentTopic.getName()).getNamespaceObject(); for (Binding binding : this.bindings) { + // The initialization queue triggers the message expiration detection task + if ("queue".equals(binding.desType)) { + futures.add(amqpAdmin.loadQueue(namespaceName, binding.des)); + } this.exchangeMessageRouter.addBinding(binding.des, binding.desType, binding.key, binding.arguments); } - this.exchangeMessageRouter.start(); + FutureUtil.waitForAll(futures).whenComplete((__, throwable) -> { + if (throwable != null) { + log.error("Failed to init queue [{}]", exchangeName, throwable); + } + this.exchangeMessageRouter.start(); + }); return; } @@ -302,7 +323,7 @@ public void removeQueue(AmqpQueue queue) { } @Override - public Topic getTopic(){ + public Topic getTopic() { return persistentTopic; } @@ -337,21 +358,21 @@ private CompletableFuture createCursorIfNotExists(String name) { } ledger.asyncOpenCursor(name, CommandSubscribe.InitialPosition.Earliest, new AsyncCallbacks.OpenCursorCallback() { - @Override - public void openCursorComplete(ManagedCursor cursor, Object ctx) { - cursorFuture.complete(cursor); - } + @Override + public void openCursorComplete(ManagedCursor cursor, Object ctx) { + cursorFuture.complete(cursor); + } - @Override - public void openCursorFailed(ManagedLedgerException exception, Object ctx) { - log.error("[{}] Failed to open cursor. ", name, exception); - cursorFuture.completeExceptionally(exception); - if (cursors.get(name) != null && cursors.get(name).isCompletedExceptionally() - || cursors.get(name).isCancelled()) { - cursors.remove(name); + @Override + public void openCursorFailed(ManagedLedgerException exception, Object ctx) { + log.error("[{}] Failed to open cursor. ", name, exception); + cursorFuture.completeExceptionally(exception); + if (cursors.get(name) != null && cursors.get(name).isCompletedExceptionally() + || cursors.get(name).isCancelled()) { + cursors.remove(name); + } } - } - }, null); + }, null); return cursorFuture; }); } @@ -397,7 +418,7 @@ public void topicNameValidate() { @Override public CompletableFuture queueBind(String queue, String routingKey, Map arguments) { - this.bindings.add(new Binding(queue, "queue", routingKey, arguments)); + this.bindings.add(new Binding(queue, "queue", routingKey, exchangeName, arguments)); String bindingsJson; try { bindingsJson = JSON_MAPPER.writeValueAsString(this.bindings); @@ -408,24 +429,24 @@ public CompletableFuture queueBind(String queue, String routingKey, Map future = new CompletableFuture<>(); this.persistentTopic.getManagedLedger().asyncSetProperty(BINDINGS, bindingsJson, new AsyncCallbacks.UpdatePropertiesCallback() { - @Override - public void updatePropertiesComplete(Map properties, Object ctx) { - PersistentExchange.this.exchangeMessageRouter.addBinding(queue, "queue", routingKey, arguments); - future.complete(null); - } + @Override + public void updatePropertiesComplete(Map properties, Object ctx) { + PersistentExchange.this.exchangeMessageRouter.addBinding(queue, "queue", routingKey, arguments); + future.complete(null); + } - @Override - public void updatePropertiesFailed(ManagedLedgerException exception, Object ctx) { - log.error("Failed to save binding metadata for bind operation.", exception); - future.completeExceptionally(exception); - } - }, null); + @Override + public void updatePropertiesFailed(ManagedLedgerException exception, Object ctx) { + log.error("Failed to save binding metadata for bind operation.", exception); + future.completeExceptionally(exception); + } + }, null); return future; } @Override public CompletableFuture queueUnBind(String queue, String routingKey, Map arguments) { - this.bindings.remove(new Binding(queue, "queue", routingKey, arguments)); + this.bindings.remove(new Binding(queue, "queue", routingKey, exchangeName, arguments)); String bindingsJson; try { bindingsJson = JSON_MAPPER.writeValueAsString(this.bindings); @@ -452,4 +473,10 @@ public void updatePropertiesFailed(ManagedLedgerException exception, Object ctx) return future; } + @Override + public void close() { + if (exchangeMessageRouter != null) { + exchangeMessageRouter.close(); + } + } } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/impl/PersistentQueue.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/impl/PersistentQueue.java index 3d9fdf9f..c90f761f 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/impl/PersistentQueue.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/impl/PersistentQueue.java @@ -3,7 +3,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,37 +13,67 @@ */ package io.streamnative.pulsar.handlers.amqp.impl; +import static io.streamnative.pulsar.handlers.amqp.utils.TopicUtil.getTopicName; +import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.FALSE; +import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.TRUE; import static org.apache.curator.shaded.com.google.common.base.Preconditions.checkArgument; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import io.netty.buffer.ByteBuf; +import io.netty.util.ReferenceCountUtil; import io.streamnative.pulsar.handlers.amqp.AbstractAmqpMessageRouter; import io.streamnative.pulsar.handlers.amqp.AbstractAmqpQueue; import io.streamnative.pulsar.handlers.amqp.AmqpEntryWriter; import io.streamnative.pulsar.handlers.amqp.AmqpExchange; import io.streamnative.pulsar.handlers.amqp.AmqpMessageRouter; +import io.streamnative.pulsar.handlers.amqp.AmqpProtocolHandler; import io.streamnative.pulsar.handlers.amqp.AmqpQueueProperties; import io.streamnative.pulsar.handlers.amqp.ExchangeContainer; import io.streamnative.pulsar.handlers.amqp.IndexMessage; +import io.streamnative.pulsar.handlers.amqp.common.exception.AoPServiceRuntimeException; import io.streamnative.pulsar.handlers.amqp.utils.MessageConvertUtils; import io.streamnative.pulsar.handlers.amqp.utils.PulsarTopicMetadataUtils; +import io.streamnative.pulsar.handlers.amqp.utils.QueueUtil; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.WaitingEntryCallBack; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.MessageImpl; +import org.apache.pulsar.client.impl.ProducerImpl; +import org.apache.pulsar.common.api.proto.CommandSubscribe; +import org.apache.pulsar.common.api.proto.KeyValue; +import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.util.FutureUtil; +import org.jetbrains.annotations.NotNull; /** * Persistent queue. @@ -53,24 +83,328 @@ public class PersistentQueue extends AbstractAmqpQueue { public static final String QUEUE = "QUEUE"; public static final String ROUTERS = "ROUTERS"; public static final String TOPIC_PREFIX = "__amqp_queue__"; + public static final String DURABLE = "DURABLE"; + public static final String PASSIVE = "PASSIVE"; + public static final String EXCLUSIVE = "EXCLUSIVE"; + public static final String AUTO_DELETE = "AUTO_DELETE"; + public static final String INTERNAL = "INTERNAL"; + public static final String ARGUMENTS = "ARGUMENTS"; + public static final String X_DEAD_LETTER_EXCHANGE = "x-dead-letter-exchange"; + public static final String X_MESSAGE_TTL = "x-message-ttl"; + public static final String X_DEAD_LETTER_ROUTING_KEY = "x-dead-letter-routing-key"; + public static final String DEFAULT_SUBSCRIPTION = "AMQP_DEFAULT"; + public static final long DELAY_1000 = 1000; + public static final long MAX_TTL = 50L * 24 * 60 * 60 * 1000; @Getter - private PersistentTopic indexTopic; + private final PersistentTopic indexTopic; - private ObjectMapper jsonMapper; + private final ObjectMapper jsonMapper; private AmqpEntryWriter amqpEntryWriter; + private CompletableFuture> deadLetterProducer; + private String deadLetterExchange; + private long queueMessageTtl; + private String deadLetterRoutingKey; + private PersistentSubscription defaultSubscription; + + private final ScheduledExecutorService scheduledExecutor; + + private volatile int isActive = FALSE; + + private static final AtomicIntegerFieldUpdater ACTIVE_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(PersistentQueue.class, "isActive"); + + private volatile int isWaiting = FALSE; + + private static final AtomicIntegerFieldUpdater WAITING_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(PersistentQueue.class, "isWaiting"); + + private volatile int retry = 0; + + private static final AtomicIntegerFieldUpdater RETRY_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(PersistentQueue.class, "retry"); + public PersistentQueue(String queueName, PersistentTopic indexTopic, long connectionId, - boolean exclusive, boolean autoDelete) { - super(queueName, true, connectionId, exclusive, autoDelete); + boolean exclusive, boolean autoDelete, Map properties) { + super(queueName, true, connectionId, exclusive, autoDelete, properties); this.indexTopic = indexTopic; + this.scheduledExecutor = indexTopic.getBrokerService().executor(); topicNameValidate(); this.jsonMapper = new ObjectMapper(); this.amqpEntryWriter = new AmqpEntryWriter(indexTopic); } + private CompletableFuture initMessageExpire() { + String args = properties.get(ARGUMENTS); + if (StringUtils.isNotBlank(args)) { + arguments.putAll(QueueUtil.covertStringValueAsObjectMap(args)); + this.deadLetterExchange = (String) arguments.get(X_DEAD_LETTER_EXCHANGE); + Object messageTtl = arguments.get(X_MESSAGE_TTL); + if (messageTtl != null && NumberUtils.isCreatable(messageTtl.toString())) { + this.queueMessageTtl = NumberUtils.createLong(messageTtl.toString()); + } + this.deadLetterRoutingKey = (String) arguments.get(X_DEAD_LETTER_ROUTING_KEY); + + if (StringUtils.isNotBlank(deadLetterExchange)) { + // init producer + if (StringUtils.isBlank(deadLetterRoutingKey)) { + this.deadLetterRoutingKey = ""; + } + NamespaceName namespaceName = TopicName.get(indexTopic.getName()).getNamespaceObject(); + String topic = getTopicName(PersistentExchange.TOPIC_PREFIX, + namespaceName.getTenant(), namespaceName.getLocalName(), deadLetterExchange); + if (indexTopic.getBrokerService().getPulsar().getProtocolHandlers() + .protocol("amqp") instanceof AmqpProtocolHandler protocolHandler) { + return protocolHandler.getAmqpBrokerService().getAmqpAdmin() + .loadExchange(namespaceName, deadLetterExchange) + .thenCompose(__ -> this.deadLetterProducer = initDeadLetterProducer(indexTopic, topic)) + .thenApply(__-> null); + } + } + } + return CompletableFuture.completedFuture(null); + } + + public CompletableFuture startMessageExpireChecker() { + if (ACTIVE_UPDATER.compareAndSet(this, FALSE, TRUE)) { + return initMessageExpire() + .thenCompose(__-> initDefaultSubscription()) + .thenAcceptAsync(subscription -> { + RETRY_UPDATER.set(this, 0); + this.defaultSubscription = (PersistentSubscription) subscription; + // start check expired + readEntries(); + log.info("[{}] Message expiration checker started successfully", indexTopic.getName()); + }).exceptionally(throwable -> { + log.warn("Retry count:{} Queue {} DQL is not created, DQL: {} ,ex {}", + retry, queueName, deadLetterExchange, throwable); + ACTIVE_UPDATER.compareAndSet(this, TRUE, FALSE); + // TODO Application alarm notification needs to be added + scheduledExecutor.schedule(this::startMessageExpireChecker, 10 * DELAY_1000, TimeUnit.MILLISECONDS); + return null; + }); + } + return CompletableFuture.completedFuture(null); + } + + public CompletableFuture initDefaultSubscription() { + Subscription subscription = indexTopic.getSubscriptions().get(DEFAULT_SUBSCRIPTION); + if (subscription != null) { + return CompletableFuture.completedFuture(subscription); + } + return indexTopic.createSubscription(DEFAULT_SUBSCRIPTION, CommandSubscribe.InitialPosition.Earliest, false, + Collections.emptyMap()); + } + + class WaitingCallBack implements WaitingEntryCallBack { + + public WaitingCallBack() { + } + + @Override + public void entriesAvailable() { + if (isActive == TRUE && WAITING_UPDATER.compareAndSet(PersistentQueue.this, TRUE, FALSE)) { + PersistentQueue.this.scheduledExecutor.execute(PersistentQueue.this::readEntries); + } + } + } + + private void readEntries() { + if (isActive == FALSE) { + return; + } + // 1. If there are active consumers, stop monitoring + // 2. Start detection when the consumer is closed and enter the ledger's wait queue + // 3. When the waiting queue is woken up, the detection again detects whether there is a consumer. If there is + // no consumer, then read it. + if (defaultSubscription.getDispatcher() != null && defaultSubscription.getDispatcher() + .isConsumerConnected()) { + log.warn("[{}] There are active consumers to stop monitoring", queueName); + ACTIVE_UPDATER.set(this, FALSE); + return; + } + ManagedCursor cursor = defaultSubscription.getCursor(); + if (defaultSubscription.getNumberOfEntriesInBacklog(false) == 0) { + if (cursor.getManagedLedger() instanceof ManagedLedgerImpl managedLedger) { + if (managedLedger.isTerminated()) { + log.warn("[{}]ledger is close", queueName); + ACTIVE_UPDATER.set(this, FALSE); + return; + } + log.warn("[{}]start waiting read.", queueName); + if (WAITING_UPDATER.compareAndSet(this, FALSE, TRUE)) { + managedLedger.addWaitingEntryCallBack(new WaitingCallBack()); + } + } else { + scheduledExecutor.schedule(PersistentQueue.this::readEntries, 5 * DELAY_1000, TimeUnit.MILLISECONDS); + } + return; + } + // To send a message to the topic, the cursor is not read from the first, need to reset + cursor.rewind(); + // Use cursor.asyncReadEntriesOrWait() can register only one wait.The user's consumer startup will fail. + cursor.asyncReadEntries(1, new AsyncCallbacks.ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + if (entries.size() == 0) { + log.warn("[{}] read entries is 0, need retry", queueName); + scheduledExecutor.execute(PersistentQueue.this::readEntries); + return; + } + Entry entry = entries.get(0); + ByteBuf dataBuffer = entry.getDataBuffer(); + Position expirePosition = entry.getPosition(); + MessageMetadata messageMetadata = Commands.parseMessageMetadata(dataBuffer); + try { + // queue ttl + long expireTime = queueMessageTtl; + // message ttl + KeyValue keyValue = messageMetadata.getPropertiesList().stream() + .filter(kv -> MessageConvertUtils.PROP_EXPIRATION.equals(kv.getKey())) + .findFirst() + .orElse(null); + long messageTtl; + if (keyValue != null && (messageTtl = Long.parseLong(keyValue.getValue())) > 0) { + expireTime = expireTime == 0 ? messageTtl : Math.min(expireTime, messageTtl); + } + // In most cases, the TTL is not available and the check task needs to be stopped. + if (expireTime == 0) { + // It is possible to mix expired messages with non-expired messages + // Stop check + // Sending a non-TTL message requires the presence of a consumer message. When a consumer + // exists, the current task will not be executed here. + ACTIVE_UPDATER.set(PersistentQueue.this, FALSE); + log.warn("[{}] Queue message TTL is not set, stop check trace", queueName); + return; + } + long expireMillis; + // no expire + if ((expireMillis = entryExpired(expireTime, messageMetadata.getPublishTime())) > 0) { + if (expireTime >= MAX_TTL) { + log.warn("[{}]There is a message with a very long expiration time {}.", + defaultSubscription.getTopic().getName(), expireTime); + try { + indexTopic.getBrokerService() + .pulsar() + .getAdminClient() + .topics() + .setMessageTTL(defaultSubscription.getTopic().getName(), + (int) (expireTime / 1000 + 24 * 60 * 60)); + } catch (Exception e) { + log.error("[{}] Failed to reset topic ttl:{}.", + defaultSubscription.getTopic().getName(), expireTime); + // TODO Application alarm notification needs to be added + } + } + // The current topic may have been unloaded, isActive=false, when the task is awakened, filter + // the task through the isActive check + scheduledExecutor.schedule(PersistentQueue.this::readEntries, expireMillis, + TimeUnit.MILLISECONDS); + return; + } + // expire but no dead letter queue + if (deadLetterProducer == null) { + log.warn("Message expired, no dead-letter-producer, [{}] message auto ack [{}]", queueName, + expirePosition); + // ack + makeAck(expirePosition, cursor).thenRun( + () -> scheduledExecutor.execute(PersistentQueue.this::readEntries)) + .exceptionally(throwable -> { + log.error("no dead-letter-producer ack fail", throwable); + scheduledExecutor.schedule(PersistentQueue.this::readEntries, 5 * DELAY_1000, + TimeUnit.MILLISECONDS); + return null; + }); + return; + } + dataBuffer.retain(); + } finally { + entry.release(); + } + messageMetadata.clearSequenceId(); + messageMetadata.clearPublishTime(); + messageMetadata.clearProducerName(); + messageMetadata.getPropertiesList().forEach(kv -> { + switch (kv.getKey()) { + case MessageConvertUtils.PROP_ROUTING_KEY -> kv.setValue(deadLetterRoutingKey); + case MessageConvertUtils.PROP_EXPIRATION -> kv.setValue("0"); + case MessageConvertUtils.PROP_EXCHANGE -> kv.setValue(deadLetterExchange); + default -> { + } + } + }); + final int readerIndex = dataBuffer.readerIndex(); + dataBuffer.readerIndex(readerIndex); + deadLetterProducer.thenCompose(producer -> { + MessageImpl message = MessageImpl.create(null, null, messageMetadata, + dataBuffer, Optional.empty(), null, Schema.BYTES, + 0, false, -1L); + return ((ProducerImpl) producer).sendAsync(message); + }) + .thenAccept(__ -> makeAck(expirePosition, cursor)) + .thenRun(() -> { + // Read the next one immediately + scheduledExecutor.execute(PersistentQueue.this::readEntries); + }) + .whenComplete((__, throwable) -> { + ReferenceCountUtil.safeRelease(dataBuffer); + if (throwable != null) { + log.error("[{}] Failed to send a dead letter queue", queueName, throwable); + scheduledExecutor.schedule(PersistentQueue.this::readEntries, 5 * DELAY_1000, + TimeUnit.MILLISECONDS); + } + }); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + log.error("[{}]Failed to read entries", queueName, exception); + scheduledExecutor.schedule(PersistentQueue.this::readEntries, 5 * DELAY_1000, TimeUnit.MILLISECONDS); + } + }, null, null); + } + + @NotNull + private CompletableFuture makeAck(Position position, ManagedCursor cursor) { + CompletableFuture future = new CompletableFuture<>(); + cursor.asyncDelete(position, new AsyncCallbacks.DeleteCallback() { + @Override + public void deleteComplete(Object ctx) { + future.complete(null); + } + + @Override + public void deleteFailed(ManagedLedgerException exception, + Object ctx) { + log.error("[{}] Message expired to delete exception, position {}", queueName, + position, exception); + future.completeExceptionally(exception); + } + }, position); + return future; + } + + public static long entryExpired(long expireMillis, long entryTimestamp) { + return (entryTimestamp + expireMillis) - System.currentTimeMillis(); + } + + private CompletableFuture> initDeadLetterProducer(PersistentTopic indexTopic, String topic) { + try { + return indexTopic.getBrokerService().pulsar() + .getClient() + .newProducer() + .topic(topic) + .enableBatching(false) + .createAsync(); + } catch (Exception e) { + log.error("init dead letter producer fail", e); + throw new AoPServiceRuntimeException.ProducerCreationRuntimeException(e); + } + } + @Override public CompletableFuture writeIndexMessageAsync(String exchangeName, long ledgerId, long entryId, Map properties) { @@ -97,7 +431,7 @@ public CompletableFuture acknowledgeAsync(String exchangeName, long ledger @Override public CompletableFuture bindExchange(AmqpExchange exchange, AmqpMessageRouter router, String bindingKey, - Map arguments) { + Map arguments) { return super.bindExchange(exchange, router, bindingKey, arguments).thenApply(__ -> { updateQueueProperties(); return null; @@ -186,4 +520,12 @@ private void topicNameValidate() { TOPIC_PREFIX, "exchangeName"); } + @Override + public void close() { + ACTIVE_UPDATER.set(this, FALSE); + if (deadLetterProducer != null) { + deadLetterProducer.thenAccept(Producer::closeAsync); + } + } + } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/utils/HttpUtil.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/utils/HttpUtil.java index 16aa5c15..8c03c59e 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/utils/HttpUtil.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/utils/HttpUtil.java @@ -14,11 +14,18 @@ package io.streamnative.pulsar.handlers.amqp.utils; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.collect.Maps; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.concurrent.CompletableFuture; +import lombok.extern.slf4j.Slf4j; import okhttp3.Call; import okhttp3.Callback; +import okhttp3.Headers; import okhttp3.MediaType; import okhttp3.OkHttpClient; import okhttp3.Request; @@ -29,13 +36,106 @@ /** * HttpUtil. */ +@Slf4j public class HttpUtil { private static final OkHttpClient client = new OkHttpClient(); private static final MediaType JSON = MediaType.get("application/json; charset=utf-8"); + public static CompletableFuture getAsync(String url, Class classType){ + return getAsync(url, new HashMap<>(), classType); + } + + public static CompletableFuture getAsync(String url, Map headers) { + Request request = new Request.Builder() + .url(url) + .headers(Headers.of(headers)) + .get() + .build(); + + CompletableFuture future = new CompletableFuture<>(); + client.newCall(request).enqueue(new Callback() { + @Override + public void onFailure(@NotNull Call call, @NotNull IOException e) { + future.completeExceptionally(e); + } + + @Override + public void onResponse(@NotNull Call call, @NotNull Response response) { + if (!response.isSuccessful()) { + future.completeExceptionally(new IOException("Unexpected code " + response)); + return; + } + future.complete(null); + } + }); + return future; + } + + public static CompletableFuture getAsync(String url, Map headers, Class classType) { + Request request = new Request.Builder() + .url(url) + .headers(Headers.of(headers)) + .get() + .build(); + + CompletableFuture future = new CompletableFuture<>(); + client.newCall(request).enqueue(new Callback() { + @Override + public void onFailure(@NotNull Call call, @NotNull IOException e) { + future.completeExceptionally(e); + } + + @Override + public void onResponse(@NotNull Call call, @NotNull Response response) throws IOException { + if (!response.isSuccessful()) { + future.completeExceptionally(new IOException("Unexpected code " + response)); + return; + } + byte[] bytes = Objects.requireNonNull(response.body()).bytes(); + T metricsResponse = + JsonUtil.parseObject(new String(bytes, StandardCharsets.UTF_8), classType); + future.complete(metricsResponse); + } + }); + return future; + } + + public static CompletableFuture getAsync(String url, Map headers, TypeReference typeReference) { + Request request = new Request.Builder() + .url(url) + .headers(Headers.of(headers)) + .get() + .build(); + + CompletableFuture future = new CompletableFuture<>(); + client.newCall(request).enqueue(new Callback() { + @Override + public void onFailure(@NotNull Call call, @NotNull IOException e) { + future.completeExceptionally(e); + } + + @Override + public void onResponse(@NotNull Call call, @NotNull Response response) throws IOException { + if (!response.isSuccessful()) { + future.completeExceptionally(new IOException("Unexpected code " + response)); + return; + } + byte[] bytes = Objects.requireNonNull(response.body()).bytes(); + T metricsResponse = + JsonUtil.parseObject(new String(bytes, StandardCharsets.UTF_8), typeReference); + future.complete(metricsResponse); + } + }); + return future; + } + public static CompletableFuture putAsync(String url, Map params) { + return putAsync(url, params, Maps.newHashMap()); + } + + public static CompletableFuture putAsync(String url, Map params, Map headers) { RequestBody requestBody; try { requestBody = RequestBody.create(JsonUtil.toString(params), JSON); @@ -44,6 +144,7 @@ public static CompletableFuture putAsync(String url, Map p } Request request = new Request.Builder() .url(url) + .headers(Headers.of(headers)) .put(requestBody) .build(); @@ -66,7 +167,11 @@ public void onResponse(@NotNull Call call, @NotNull Response response) throws IO return future; } - public static CompletableFuture postAsync(String url, Map params) { + public static CompletableFuture postAsync(String url, Map params){ + return postAsync(url, params, Maps.newHashMap()); + } + + public static CompletableFuture postAsync(String url, Map params, Map headers) { RequestBody requestBody; try { requestBody = RequestBody.create(JsonUtil.toString(params), JSON); @@ -75,6 +180,7 @@ public static CompletableFuture postAsync(String url, Map } Request request = new Request.Builder() .url(url) + .headers(Headers.of(headers)) .post(requestBody) .build(); @@ -96,8 +202,10 @@ public void onResponse(@NotNull Call call, @NotNull Response response) throws IO }); return future; } - - public static CompletableFuture deleteAsync(String url, Map params) { + public static CompletableFuture deleteAsync(String url, Map params){ + return deleteAsync(url, params, Maps.newHashMap()); + } + public static CompletableFuture deleteAsync(String url, Map params, Map headers) { RequestBody requestBody; try { requestBody = RequestBody.create(JsonUtil.toString(params), JSON); @@ -106,6 +214,7 @@ public static CompletableFuture deleteAsync(String url, Map T parseObject(String jsonStr) throws IOException { public static T parseObject(String jsonStr, Class clazz) throws IOException { return objectMapper.readValue(jsonStr, clazz); } + public static T parseObject(String jsonStr, TypeReference tTypeReference) throws IOException { + return objectMapper.readValue(jsonStr, tTypeReference); + } public static List parseObjectList(String json, Class obj) throws JsonProcessingException { JavaType javaType = objectMapper.getTypeFactory().constructParametricType(ArrayList.class, obj); diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/utils/MessageConvertUtils.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/utils/MessageConvertUtils.java index b677ed29..c067bd57 100644 --- a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/utils/MessageConvertUtils.java +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/utils/MessageConvertUtils.java @@ -3,7 +3,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,22 +15,26 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; - import com.google.common.collect.ImmutableList; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.streamnative.pulsar.handlers.amqp.AmqpMessageData; import io.streamnative.pulsar.handlers.amqp.IndexMessage; +import io.streamnative.pulsar.handlers.amqp.admin.model.PublishParams; +import io.streamnative.pulsar.handlers.amqp.common.exception.AoPServiceRuntimeException; import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.time.Clock; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.TimeUnit; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Entry; +import org.apache.commons.lang3.math.NumberUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.Message; @@ -38,6 +42,7 @@ import org.apache.pulsar.client.api.TypedMessageBuilder; import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.client.impl.TypedMessageBuilderImpl; +import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.api.proto.KeyValue; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.compression.CompressionCodecProvider; @@ -45,6 +50,7 @@ import org.apache.qpid.server.bytebuffer.QpidByteBuffer; import org.apache.qpid.server.bytebuffer.SingleQpidByteBuffer; import org.apache.qpid.server.protocol.v0_8.AMQShortString; +import org.apache.qpid.server.protocol.v0_8.FieldTable; import org.apache.qpid.server.protocol.v0_8.FieldTableFactory; import org.apache.qpid.server.protocol.v0_8.IncomingMessage; import org.apache.qpid.server.protocol.v0_8.transport.BasicContentHeaderProperties; @@ -62,17 +68,17 @@ public final class MessageConvertUtils { private static final String FAKE_AMQP_PRODUCER_NAME = "fake_amqp_producer_name"; private static final String DEFAULT_CHARSET_NAME = "ISO8859-1"; private static final String PROP_DELIMITER = "."; - private static final String BASIC_PROP_PRE = "_bp_" + PROP_DELIMITER; - private static final String BASIC_PROP_HEADER_PRE = "_bph_" + PROP_DELIMITER; + public static final String BASIC_PROP_PRE = "_bp_" + PROP_DELIMITER; + public static final String BASIC_PROP_HEADER_PRE = "_bph_" + PROP_DELIMITER; private static final String BASIC_PUBLISH_INFO_PRE = "_pi_" + PROP_DELIMITER; - private static final String PROP_CONTENT_TYPE = BASIC_PROP_PRE + "content_type"; - private static final String PROP_ENCODING = BASIC_PROP_PRE + "encoding"; + public static final String PROP_CONTENT_TYPE = BASIC_PROP_PRE + "content_type"; + public static final String PROP_ENCODING = BASIC_PROP_PRE + "encoding"; private static final String PROP_DELIVERY_MODE = BASIC_PROP_PRE + "delivery_mode"; - private static final String PROP_PRIORITY_PRIORITY = BASIC_PROP_PRE + "priority"; + public static final String PROP_PRIORITY_PRIORITY = BASIC_PROP_PRE + "priority"; private static final String PROP_CORRELATION_ID = BASIC_PROP_PRE + "correlation_id"; private static final String PROP_REPLY_TO = BASIC_PROP_PRE + "reply_to"; - private static final String PROP_EXPIRATION = BASIC_PROP_PRE + "expiration"; + public static final String PROP_EXPIRATION = BASIC_PROP_PRE + "expiration"; private static final String PROP_MESSAGE_ID = BASIC_PROP_PRE + "message_id"; private static final String PROP_TIMESTAMP = BASIC_PROP_PRE + "timestamp"; private static final String PROP_TYPE = BASIC_PROP_PRE + "type"; @@ -81,37 +87,52 @@ public final class MessageConvertUtils { private static final String PROP_CLUSTER_ID = BASIC_PROP_PRE + "cluster_id"; private static final String PROP_PROPERTY_FLAGS = BASIC_PROP_PRE + "property_flags"; - private static final String PROP_EXCHANGE = BASIC_PUBLISH_INFO_PRE + "exchange"; + public static final String PROP_EXCHANGE = BASIC_PUBLISH_INFO_PRE + "exchange"; private static final String PROP_IMMEDIATE = BASIC_PUBLISH_INFO_PRE + "immediate"; private static final String PROP_MANDATORY = BASIC_PUBLISH_INFO_PRE + "mandatory"; public static final String PROP_ROUTING_KEY = BASIC_PUBLISH_INFO_PRE + "routingKey"; + public static final String BASIC_PROP_HEADER_X_DELAY = BASIC_PROP_HEADER_PRE + "x-delay"; private static final Clock clock = Clock.systemDefaultZone(); // convert qpid IncomingMessage to Pulsar MessageImpl public static MessageImpl toPulsarMessage(IncomingMessage incomingMessage) throws UnsupportedEncodingException { - @SuppressWarnings("unchecked") - TypedMessageBuilderImpl builder = new TypedMessageBuilderImpl(null, Schema.BYTES); - + MessageImpl message; // value if (incomingMessage.getBodyCount() > 0) { - ByteBuf byteBuf = Unpooled.buffer(); + ByteBuf byteBuf = PulsarByteBufAllocator.DEFAULT.buffer((int)incomingMessage.getContentHeader().getBodySize()); for (int i = 0; i < incomingMessage.getBodyCount(); i++) { - byteBuf.writeBytes( - ((SingleQpidByteBuffer) incomingMessage.getContentChunk(i).getPayload()) - .getUnderlyingBuffer()); + SingleQpidByteBuffer payload = (SingleQpidByteBuffer) incomingMessage.getContentChunk(i).getPayload(); + byteBuf.writeBytes(payload.getUnderlyingBuffer()); + payload.dispose(); } - byte[] bytes = new byte[byteBuf.writerIndex()]; - byteBuf.readBytes(bytes, 0, byteBuf.writerIndex()); - builder.value(bytes); + message = MessageImpl.create(null, null, new MessageMetadata(), byteBuf, + Optional.empty(), null, Schema.BYTES, 0, true, -1L); + byteBuf.release(); } else { - builder.value(new byte[0]); + message = MessageImpl.create(null, null, new MessageMetadata(), Unpooled.EMPTY_BUFFER, + Optional.empty(), null, Schema.BYTES, 0, false, -1L); } - + MessageMetadata metadata = message.getMessageBuilder(); // basic properties ContentHeaderBody contentHeaderBody = incomingMessage.getContentHeader(); BasicContentHeaderProperties props = contentHeaderBody.getProperties(); + try { + setProp(metadata, props); + setProp(metadata, PROP_EXCHANGE, incomingMessage.getMessagePublishInfo().getExchange()); + setProp(metadata, PROP_IMMEDIATE, incomingMessage.getMessagePublishInfo().isImmediate()); + setProp(metadata, PROP_MANDATORY, incomingMessage.getMessagePublishInfo().isMandatory()); + setProp(metadata, PROP_ROUTING_KEY, incomingMessage.getMessagePublishInfo().getRoutingKey()); + } catch (UnsupportedEncodingException e) { + message.release(); + throw e; + } + return message; + } + + public static void setProp(TypedMessageBuilderImpl builder, BasicContentHeaderProperties props) + throws UnsupportedEncodingException { if (props != null) { if (props.getTimestamp() > 0) { builder.eventTime(props.getTimestamp()); @@ -119,8 +140,8 @@ public static MessageImpl toPulsarMessage(IncomingMessage incomingMessag setProp(builder, PROP_CONTENT_TYPE, props.getContentTypeAsString()); setProp(builder, PROP_ENCODING, props.getEncodingAsString()); - setProp(builder, PROP_DELIVERY_MODE, props.getDeliveryMode()); - setProp(builder, PROP_PRIORITY_PRIORITY, props.getPriority()); + setProp(builder, PROP_DELIVERY_MODE, ((Byte) props.getDeliveryMode()).intValue()); + setProp(builder, PROP_PRIORITY_PRIORITY, ((Byte) props.getPriority()).intValue()); setProp(builder, PROP_CORRELATION_ID, props.getCorrelationIdAsString()); setProp(builder, PROP_REPLY_TO, props.getReplyToAsString()); setProp(builder, PROP_EXPIRATION, props.getExpiration()); @@ -137,16 +158,75 @@ public static MessageImpl toPulsarMessage(IncomingMessage incomingMessag setProp(builder, BASIC_PROP_HEADER_PRE + entry.getKey(), entry.getValue()); } } + } - setProp(builder, PROP_EXCHANGE, incomingMessage.getMessagePublishInfo().getExchange()); - setProp(builder, PROP_IMMEDIATE, incomingMessage.getMessagePublishInfo().isImmediate()); - setProp(builder, PROP_MANDATORY, incomingMessage.getMessagePublishInfo().isMandatory()); - setProp(builder, PROP_ROUTING_KEY, incomingMessage.getMessagePublishInfo().getRoutingKey()); + public static void setProp(MessageMetadata msgMetadata, BasicContentHeaderProperties props) + throws UnsupportedEncodingException { + if (props != null) { + if (props.getTimestamp() > 0) { + msgMetadata.setEventTime(props.getTimestamp()); + } + + setProp(msgMetadata, PROP_CONTENT_TYPE, props.getContentTypeAsString()); + setProp(msgMetadata, PROP_ENCODING, props.getEncodingAsString()); + setProp(msgMetadata, PROP_DELIVERY_MODE, ((Byte) props.getDeliveryMode()).intValue()); + setProp(msgMetadata, PROP_PRIORITY_PRIORITY, ((Byte) props.getPriority()).intValue()); + setProp(msgMetadata, PROP_CORRELATION_ID, props.getCorrelationIdAsString()); + setProp(msgMetadata, PROP_REPLY_TO, props.getReplyToAsString()); + setProp(msgMetadata, PROP_EXPIRATION, props.getExpiration()); + setProp(msgMetadata, PROP_MESSAGE_ID, props.getMessageIdAsString()); + setProp(msgMetadata, PROP_TIMESTAMP, props.getTimestamp()); + setProp(msgMetadata, PROP_TYPE, props.getTypeAsString()); + setProp(msgMetadata, PROP_USER_ID, props.getUserIdAsString()); + setProp(msgMetadata, PROP_APP_ID, props.getAppIdAsString()); + setProp(msgMetadata, PROP_CLUSTER_ID, props.getClusterIdAsString()); + setProp(msgMetadata, PROP_PROPERTY_FLAGS, props.getPropertyFlags()); + Map headers = props.getHeadersAsMap(); + for (Map.Entry entry : headers.entrySet()) { + setProp(msgMetadata, BASIC_PROP_HEADER_PRE + entry.getKey(), entry.getValue()); + } + } + } + + public static MessageImpl toPulsarMessage(PublishParams params) { + TypedMessageBuilderImpl builder = new TypedMessageBuilderImpl<>(null, Schema.BYTES); + BasicContentHeaderProperties properties = new BasicContentHeaderProperties(); + properties.setDeliveryMode(Byte.parseByte(params.getDeliveryMode())); + properties.setHeaders(FieldTable.convertToFieldTable(params.getHeaders())); + properties.setContentType(params.getProps().get("content_type")); + String expiration = params.getProps().get("expiration"); + if (NumberUtils.isCreatable(expiration)) { + properties.setExpiration(Long.parseLong(expiration)); + } + properties.setEncoding(params.getProps().get("content_encoding")); + String priority = params.getProps().get("priority"); + if (NumberUtils.isCreatable(priority)) { + properties.setPriority(Byte.parseByte(priority)); + } + properties.setCorrelationId(params.getProps().get("correlation_id")); + properties.setReplyTo(params.getProps().get("reply_to")); + properties.setMessageId(params.getProps().get("message_id")); + String timestamp = params.getProps().get("timestamp"); + if (NumberUtils.isCreatable(timestamp)) { + properties.setTimestamp(Long.parseLong(timestamp)); + } + properties.setType(params.getProps().get("type")); + properties.setUserId(params.getProps().get("user_id")); + properties.setAppId(params.getProps().get("app_id")); + properties.setClusterId(params.getProps().get("cluster_id")); + try { + MessageConvertUtils.setProp(builder, properties); + MessageConvertUtils.setProp(builder, MessageConvertUtils.PROP_EXCHANGE, params.getName()); + MessageConvertUtils.setProp(builder, MessageConvertUtils.PROP_ROUTING_KEY, params.getRoutingKey()); + } catch (UnsupportedEncodingException e) { + throw new AoPServiceRuntimeException.NotSupportedOperationException(e.getMessage()); + } + builder.value(params.getPayload().getBytes(StandardCharsets.UTF_8)); return (MessageImpl) builder.getMessage(); } - private void setProp(TypedMessageBuilder builder, String propName, Object value) + public static void setProp(TypedMessageBuilder builder, String propName, Object value) throws UnsupportedEncodingException { if (value != null) { if (value instanceof Byte) { @@ -157,6 +237,17 @@ private void setProp(TypedMessageBuilder builder, String propName, Object value) } } + public static void setProp(MessageMetadata metadata, String propName, Object value) + throws UnsupportedEncodingException { + if (value != null) { + if (value instanceof Byte) { + metadata.addProperty().setKey(propName).setValue(byteToString((byte) value)); + } else { + metadata.addProperty().setKey(propName).setValue(String.valueOf(value)); + } + } + } + // convert message to ByteBuf payload for ledger.addEntry. // parameter message is converted from passed in Kafka record. // called when publish received Kafka Record into Pulsar. @@ -188,7 +279,7 @@ public static ByteBuf messageToByteBuf(Message message) { } public static Pair getPropertiesFromMetadata( - List propertiesList) throws UnsupportedEncodingException { + List propertiesList) throws UnsupportedEncodingException { BasicContentHeaderProperties props = new BasicContentHeaderProperties(); Map headers = new HashMap<>(); MessagePublishInfo messagePublishInfo = new MessagePublishInfo(); @@ -328,7 +419,7 @@ public static Pair getProperti } public static List entriesToAmqpBodyList(List entries) - throws UnsupportedEncodingException { + throws UnsupportedEncodingException { ImmutableList.Builder builder = ImmutableList.builder(); // TODO convert bk entries to amqpbody, // then assemble deliver body with ContentHeaderBody and ContentBody @@ -369,7 +460,7 @@ public static List entriesToAmqpBodyList(List entries) } public static AmqpMessageData entryToAmqpBody(Entry entry) - throws UnsupportedEncodingException { + throws UnsupportedEncodingException { AmqpMessageData amqpMessage = null; // TODO convert bk entries to amqpbody, // then assemble deliver body with ContentHeaderBody and ContentBody @@ -382,9 +473,9 @@ public static AmqpMessageData entryToAmqpBody(Entry entry) try { if (log.isDebugEnabled()) { log.debug("entryToRecord. NumMessagesInBatch: {}, isBatchMessage: {}." - + " new entryId {}:{}, readerIndex: {}, writerIndex: {}", - numMessages, !notBatchMessage, entry.getLedgerId(), - entry.getEntryId(), payload.readerIndex(), payload.writerIndex()); + + " new entryId {}:{}, readerIndex: {}, writerIndex: {}", + numMessages, !notBatchMessage, entry.getLedgerId(), + entry.getEntryId(), payload.readerIndex(), payload.writerIndex()); } // need handle encryption @@ -392,7 +483,7 @@ public static AmqpMessageData entryToAmqpBody(Entry entry) if (notBatchMessage) { Pair metaData = - getPropertiesFromMetadata(msgMetadata.getPropertiesList()); + getPropertiesFromMetadata(msgMetadata.getPropertiesList()); ContentHeaderBody contentHeaderBody = new ContentHeaderBody(metaData.getLeft()); contentHeaderBody.setBodySize(payload.readableBytes()); @@ -400,10 +491,10 @@ public static AmqpMessageData entryToAmqpBody(Entry entry) byte[] data = new byte[payload.readableBytes()]; payload.readBytes(data); amqpMessage = AmqpMessageData.builder() - .messagePublishInfo(metaData.getRight()) - .contentHeaderBody(contentHeaderBody) - .contentBody(new ContentBody(QpidByteBuffer.wrap(data))) - .build(); + .messagePublishInfo(metaData.getRight()) + .contentHeaderBody(contentHeaderBody) + .contentBody(new ContentBody(QpidByteBuffer.wrap(data))) + .build(); } else { // currently, no consider for batch } @@ -415,18 +506,34 @@ public static AmqpMessageData entryToAmqpBody(Entry entry) public static AmqpMessageData messageToAmqpBody(Message message) throws UnsupportedEncodingException { - AmqpMessageData amqpMessage = null; - + AmqpMessageData amqpMessage; + Map messageProperties = new HashMap<>(message.getProperties()); + messageProperties.put(BASIC_PROP_HEADER_PRE + "pulsar_message_position", message.getMessageId().toString()); Pair metaData = - getPropertiesFromMetadata(message.getProperties()); + getPropertiesFromMetadata(messageProperties); ContentHeaderBody contentHeaderBody = new ContentHeaderBody(metaData.getLeft()); - contentHeaderBody.setBodySize(message.getData().length); + MessageImpl msg = (MessageImpl) message; + ByteBuf buf = msg.getDataBuffer(); + msg.getData(); + ByteBuffer byteBuffer; + if (buf.isDirect()) { + byteBuffer = ByteBuffer.allocateDirect(buf.readableBytes()); + buf.getBytes(buf.readerIndex(), byteBuffer); + } else if (buf.arrayOffset() == 0 && buf.capacity() == buf.array().length) { + byte[] array = buf.array(); + byteBuffer = ByteBuffer.wrap(array); + } else { + byteBuffer = ByteBuffer.allocateDirect(buf.readableBytes()); + buf.getBytes(buf.readerIndex(), byteBuffer); + } + byteBuffer.flip(); + contentHeaderBody.setBodySize(byteBuffer.limit()); amqpMessage = AmqpMessageData.builder() .messagePublishInfo(metaData.getRight()) .contentHeaderBody(contentHeaderBody) - .contentBody(new ContentBody(QpidByteBuffer.wrap(message.getData()))) + .contentBody(new ContentBody(QpidByteBuffer.wrap(byteBuffer))) .build(); return amqpMessage; } diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/utils/QueueUtil.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/utils/QueueUtil.java new file mode 100644 index 00000000..31ad7856 --- /dev/null +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/utils/QueueUtil.java @@ -0,0 +1,82 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.streamnative.pulsar.handlers.amqp.utils; + +import static io.streamnative.pulsar.handlers.amqp.impl.PersistentQueue.ARGUMENTS; +import static io.streamnative.pulsar.handlers.amqp.impl.PersistentQueue.AUTO_DELETE; +import static io.streamnative.pulsar.handlers.amqp.impl.PersistentQueue.DURABLE; +import static io.streamnative.pulsar.handlers.amqp.impl.PersistentQueue.EXCLUSIVE; +import static io.streamnative.pulsar.handlers.amqp.impl.PersistentQueue.PASSIVE; +import static io.streamnative.pulsar.handlers.amqp.impl.PersistentQueue.QUEUE; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; +import io.streamnative.pulsar.handlers.amqp.admin.model.QueueDeclareParams; +import io.streamnative.pulsar.handlers.amqp.common.exception.AoPServiceRuntimeException; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import org.apache.commons.lang3.StringUtils; + +public class QueueUtil { + + public static final ObjectMapper JSON_MAPPER = new JsonMapper(); + + public static Map generateTopicProperties(String queueName, boolean durable, boolean autoDelete, + boolean passive, Map arguments) + throws JsonProcessingException { + if (StringUtils.isEmpty(queueName)) { + throw new AoPServiceRuntimeException.ExchangeParameterException("Miss parameter queue name."); + } + Map props = new HashMap<>(8); + props.put(QUEUE, queueName); + props.put(DURABLE, "" + durable); + props.put(AUTO_DELETE, "" + autoDelete); + props.put(PASSIVE, "" + passive); + props.put(ARGUMENTS, + covertObjectValueAsString(Objects.requireNonNullElseGet(arguments, () -> new HashMap<>(1)))); + return props; + } + + public static QueueDeclareParams covertMapAsParams(Map map) { + String durable = map.get(DURABLE); + String autoDelete = map.get(AUTO_DELETE); + String exclusive = map.get(EXCLUSIVE); + String arguments = map.get(ARGUMENTS); + QueueDeclareParams params = new QueueDeclareParams(); + params.setArguments(covertStringValueAsObjectMap(arguments)); + params.setNode(""); + params.setDurable(Boolean.parseBoolean(durable)); + params.setExclusive(Boolean.parseBoolean(exclusive)); + params.setAutoDelete(Boolean.parseBoolean(autoDelete)); + return params; + } + + public static String covertObjectValueAsString(Object obj) throws JsonProcessingException { + return JSON_MAPPER.writeValueAsString(obj); + } + + public static Map covertStringValueAsObjectMap(String value) { + if (value == null || value.trim().isEmpty()) { + return new HashMap<>(); + } + try { + return JSON_MAPPER.readValue(value, new TypeReference<>() {}); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } +} diff --git a/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/utils/TopicUtil.java b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/utils/TopicUtil.java new file mode 100644 index 00000000..a98db873 --- /dev/null +++ b/amqp-impl/src/main/java/io/streamnative/pulsar/handlers/amqp/utils/TopicUtil.java @@ -0,0 +1,35 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.streamnative.pulsar.handlers.amqp.utils; + +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicDomain; + +public class TopicUtil { + + public static String getTopicName(String topicPrefix, String tenant, String namespace, String name) { + return TopicDomain.persistent + "://" + + tenant + "/" + + namespace + "/" + + topicPrefix + name; + } + + public static String getTopicName(String topicPrefix, NamespaceName namespaceName, String name) { + return TopicDomain.persistent + "://" + + namespaceName.getTenant() + "/" + + namespaceName.getLocalName() + "/" + + topicPrefix + name; + } +}