001/**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.activemq.network;
018
019import java.io.IOException;
020import java.security.GeneralSecurityException;
021import java.security.cert.X509Certificate;
022import java.util.Arrays;
023import java.util.Collection;
024import java.util.Collections;
025import java.util.Iterator;
026import java.util.List;
027import java.util.Properties;
028import java.util.Set;
029import java.util.concurrent.ConcurrentHashMap;
030import java.util.concurrent.ConcurrentMap;
031import java.util.concurrent.CountDownLatch;
032import java.util.concurrent.ExecutionException;
033import java.util.concurrent.ExecutorService;
034import java.util.concurrent.Executors;
035import java.util.concurrent.Future;
036import java.util.concurrent.TimeUnit;
037import java.util.concurrent.TimeoutException;
038import java.util.concurrent.atomic.AtomicBoolean;
039import java.util.regex.Pattern;
040
041import javax.management.ObjectName;
042
043import org.apache.activemq.DestinationDoesNotExistException;
044import org.apache.activemq.Service;
045import org.apache.activemq.advisory.AdvisoryBroker;
046import org.apache.activemq.advisory.AdvisorySupport;
047import org.apache.activemq.broker.BrokerService;
048import org.apache.activemq.broker.BrokerServiceAware;
049import org.apache.activemq.broker.ConnectionContext;
050import org.apache.activemq.broker.TransportConnection;
051import org.apache.activemq.broker.region.AbstractRegion;
052import org.apache.activemq.broker.region.DurableTopicSubscription;
053import org.apache.activemq.broker.region.Region;
054import org.apache.activemq.broker.region.RegionBroker;
055import org.apache.activemq.broker.region.Subscription;
056import org.apache.activemq.broker.region.policy.PolicyEntry;
057import org.apache.activemq.command.ActiveMQDestination;
058import org.apache.activemq.command.ActiveMQMessage;
059import org.apache.activemq.command.ActiveMQTempDestination;
060import org.apache.activemq.command.ActiveMQTopic;
061import org.apache.activemq.command.BrokerId;
062import org.apache.activemq.command.BrokerInfo;
063import org.apache.activemq.command.BrokerSubscriptionInfo;
064import org.apache.activemq.command.Command;
065import org.apache.activemq.command.CommandTypes;
066import org.apache.activemq.command.ConnectionError;
067import org.apache.activemq.command.ConnectionId;
068import org.apache.activemq.command.ConnectionInfo;
069import org.apache.activemq.command.ConsumerId;
070import org.apache.activemq.command.ConsumerInfo;
071import org.apache.activemq.command.DataStructure;
072import org.apache.activemq.command.DestinationInfo;
073import org.apache.activemq.command.ExceptionResponse;
074import org.apache.activemq.command.KeepAliveInfo;
075import org.apache.activemq.command.Message;
076import org.apache.activemq.command.MessageAck;
077import org.apache.activemq.command.MessageDispatch;
078import org.apache.activemq.command.MessageId;
079import org.apache.activemq.command.NetworkBridgeFilter;
080import org.apache.activemq.command.ProducerInfo;
081import org.apache.activemq.command.RemoveInfo;
082import org.apache.activemq.command.RemoveSubscriptionInfo;
083import org.apache.activemq.command.Response;
084import org.apache.activemq.command.SessionInfo;
085import org.apache.activemq.command.ShutdownInfo;
086import org.apache.activemq.command.SubscriptionInfo;
087import org.apache.activemq.command.WireFormatInfo;
088import org.apache.activemq.filter.DestinationFilter;
089import org.apache.activemq.filter.NonCachedMessageEvaluationContext;
090import org.apache.activemq.security.SecurityContext;
091import org.apache.activemq.transport.DefaultTransportListener;
092import org.apache.activemq.transport.FutureResponse;
093import org.apache.activemq.transport.ResponseCallback;
094import org.apache.activemq.transport.Transport;
095import org.apache.activemq.transport.TransportDisposedIOException;
096import org.apache.activemq.transport.TransportFilter;
097import org.apache.activemq.transport.failover.FailoverTransport;
098import org.apache.activemq.transport.tcp.TcpTransport;
099import org.apache.activemq.util.IdGenerator;
100import org.apache.activemq.util.IntrospectionSupport;
101import org.apache.activemq.util.LongSequenceGenerator;
102import org.apache.activemq.util.MarshallingSupport;
103import org.apache.activemq.util.NetworkBridgeUtils;
104import org.apache.activemq.util.ServiceStopper;
105import org.apache.activemq.util.ServiceSupport;
106import org.apache.activemq.util.StringToListOfActiveMQDestinationConverter;
107import org.slf4j.Logger;
108import org.slf4j.LoggerFactory;
109
110/**
111 * A useful base class for implementing demand forwarding bridges.
112 */
113public abstract class DemandForwardingBridgeSupport implements NetworkBridge, BrokerServiceAware {
114    private static final Logger LOG = LoggerFactory.getLogger(DemandForwardingBridgeSupport.class);
115    protected static final String DURABLE_SUB_PREFIX = "NC-DS_";
116    protected final Transport localBroker;
117    protected final Transport remoteBroker;
118    protected IdGenerator idGenerator = new IdGenerator();
119    protected final LongSequenceGenerator consumerIdGenerator = new LongSequenceGenerator();
120    protected ConnectionInfo localConnectionInfo;
121    protected ConnectionInfo remoteConnectionInfo;
122    protected SessionInfo localSessionInfo;
123    protected ProducerInfo producerInfo;
124    protected String remoteBrokerName = "Unknown";
125    protected String localClientId;
126    protected ConsumerInfo demandConsumerInfo;
127    protected int demandConsumerDispatched;
128    protected final AtomicBoolean localBridgeStarted = new AtomicBoolean(false);
129    protected final AtomicBoolean remoteBridgeStarted = new AtomicBoolean(false);
130    protected final AtomicBoolean bridgeFailed = new AtomicBoolean();
131    protected final AtomicBoolean disposed = new AtomicBoolean();
132    protected BrokerId localBrokerId;
133    protected ActiveMQDestination[] excludedDestinations;
134    protected ActiveMQDestination[] dynamicallyIncludedDestinations;
135    protected ActiveMQDestination[] staticallyIncludedDestinations;
136    protected ActiveMQDestination[] durableDestinations;
137    protected final ConcurrentMap<ConsumerId, DemandSubscription> subscriptionMapByLocalId = new ConcurrentHashMap<>();
138    protected final ConcurrentMap<ConsumerId, DemandSubscription> subscriptionMapByRemoteId = new ConcurrentHashMap<>();
139    protected final Set<ConsumerId> forcedDurableRemoteId = Collections.newSetFromMap(new ConcurrentHashMap<ConsumerId, Boolean>());
140    protected final BrokerId localBrokerPath[] = new BrokerId[]{null};
141    protected final CountDownLatch startedLatch = new CountDownLatch(2);
142    protected final CountDownLatch localStartedLatch = new CountDownLatch(1);
143    protected final CountDownLatch staticDestinationsLatch = new CountDownLatch(1);
144    protected final AtomicBoolean lastConnectSucceeded = new AtomicBoolean(false);
145    protected NetworkBridgeConfiguration configuration;
146    protected final NetworkBridgeFilterFactory defaultFilterFactory = new DefaultNetworkBridgeFilterFactory();
147
148    protected final BrokerId remoteBrokerPath[] = new BrokerId[]{null};
149    protected BrokerId remoteBrokerId;
150
151    protected final NetworkBridgeStatistics networkBridgeStatistics = new NetworkBridgeStatistics();
152
153    private NetworkBridgeListener networkBridgeListener;
154    private boolean createdByDuplex;
155    private BrokerInfo localBrokerInfo;
156    private BrokerInfo remoteBrokerInfo;
157
158    private final FutureBrokerInfo futureRemoteBrokerInfo = new FutureBrokerInfo(remoteBrokerInfo, disposed);
159    private final FutureBrokerInfo futureLocalBrokerInfo = new FutureBrokerInfo(localBrokerInfo, disposed);
160
161    private final AtomicBoolean started = new AtomicBoolean();
162    private TransportConnection duplexInitiatingConnection;
163    private final AtomicBoolean duplexInitiatingConnectionInfoReceived = new AtomicBoolean();
164    protected BrokerService brokerService = null;
165    private ObjectName mbeanObjectName;
166    private final ExecutorService serialExecutor = Executors.newSingleThreadExecutor();
167    //Use a new executor for processing BrokerSubscriptionInfo so we don't block other threads
168    private final ExecutorService syncExecutor = Executors.newSingleThreadExecutor();
169    private Transport duplexInboundLocalBroker = null;
170    private ProducerInfo duplexInboundLocalProducerInfo;
171
172    public DemandForwardingBridgeSupport(NetworkBridgeConfiguration configuration, Transport localBroker, Transport remoteBroker) {
173        this.configuration = configuration;
174        this.localBroker = localBroker;
175        this.remoteBroker = remoteBroker;
176    }
177
178    public void duplexStart(TransportConnection connection, BrokerInfo localBrokerInfo, BrokerInfo remoteBrokerInfo) throws Exception {
179        this.localBrokerInfo = localBrokerInfo;
180        this.remoteBrokerInfo = remoteBrokerInfo;
181        this.duplexInitiatingConnection = connection;
182        start();
183        serviceRemoteCommand(remoteBrokerInfo);
184    }
185
186    @Override
187    public void start() throws Exception {
188        if (started.compareAndSet(false, true)) {
189
190            if (brokerService == null) {
191                throw new IllegalArgumentException("BrokerService is null on " + this);
192            }
193
194            networkBridgeStatistics.setEnabled(brokerService.isEnableStatistics());
195
196            if (isDuplex()) {
197                duplexInboundLocalBroker = NetworkBridgeFactory.createLocalAsyncTransport(brokerService.getBroker().getVmConnectorURI());
198                duplexInboundLocalBroker.setTransportListener(new DefaultTransportListener() {
199
200                    @Override
201                    public void onCommand(Object o) {
202                        Command command = (Command) o;
203                        serviceLocalCommand(command);
204                    }
205
206                    @Override
207                    public void onException(IOException error) {
208                        serviceLocalException(error);
209                    }
210                });
211                duplexInboundLocalBroker.start();
212            }
213
214            localBroker.setTransportListener(new DefaultTransportListener() {
215
216                @Override
217                public void onCommand(Object o) {
218                    Command command = (Command) o;
219                    serviceLocalCommand(command);
220                }
221
222                @Override
223                public void onException(IOException error) {
224                    if (!futureLocalBrokerInfo.isDone()) {
225                        LOG.info("Error with pending local brokerInfo on: {} ({})", localBroker, error.getMessage());
226                        LOG.debug("Peer error: ", error);
227                        futureLocalBrokerInfo.cancel(true);
228                        return;
229                    }
230                    serviceLocalException(error);
231                }
232            });
233
234            remoteBroker.setTransportListener(new DefaultTransportListener() {
235
236                @Override
237                public void onCommand(Object o) {
238                    Command command = (Command) o;
239                    serviceRemoteCommand(command);
240                }
241
242                @Override
243                public void onException(IOException error) {
244                    if (!futureRemoteBrokerInfo.isDone()) {
245                        LOG.info("Error with pending remote brokerInfo on: {} ({})", remoteBroker, error.getMessage());
246                        LOG.debug("Peer error: ", error);
247                        futureRemoteBrokerInfo.cancel(true);
248                        return;
249                    }
250                    serviceRemoteException(error);
251                }
252            });
253
254            remoteBroker.start();
255            localBroker.start();
256
257            if (!disposed.get()) {
258                try {
259                    triggerStartAsyncNetworkBridgeCreation();
260                } catch (IOException e) {
261                    LOG.warn("Caught exception from remote start", e);
262                }
263            } else {
264                LOG.warn("Bridge was disposed before the start() method was fully executed.");
265                throw new TransportDisposedIOException();
266            }
267        }
268    }
269
270    @Override
271    public void stop() throws Exception {
272        if (started.compareAndSet(true, false)) {
273            if (disposed.compareAndSet(false, true)) {
274                LOG.debug(" stopping {} bridge to {}", configuration.getBrokerName(), remoteBrokerName);
275
276                futureRemoteBrokerInfo.cancel(true);
277                futureLocalBrokerInfo.cancel(true);
278
279                NetworkBridgeListener l = this.networkBridgeListener;
280                if (l != null) {
281                    l.onStop(this);
282                }
283                try {
284                    // local start complete
285                    if (startedLatch.getCount() < 2) {
286                        LOG.trace("{} unregister bridge ({}) to {}", new Object[]{
287                                configuration.getBrokerName(), this, remoteBrokerName
288                        });
289                        brokerService.getBroker().removeBroker(null, remoteBrokerInfo);
290                        brokerService.getBroker().networkBridgeStopped(remoteBrokerInfo);
291                    }
292
293                    remoteBridgeStarted.set(false);
294                    final CountDownLatch sendShutdown = new CountDownLatch(1);
295
296                    brokerService.getTaskRunnerFactory().execute(new Runnable() {
297                        @Override
298                        public void run() {
299                            try {
300                                serialExecutor.shutdown();
301                                if (!serialExecutor.awaitTermination(5, TimeUnit.SECONDS)) {
302                                    List<Runnable> pendingTasks = serialExecutor.shutdownNow();
303                                    LOG.info("pending tasks on stop {}", pendingTasks);
304                                }
305                                //Shutdown the syncExecutor, call countDown to make sure a thread can
306                                //terminate if it is waiting
307                                staticDestinationsLatch.countDown();
308                                syncExecutor.shutdown();
309                                if (!syncExecutor.awaitTermination(5, TimeUnit.SECONDS)) {
310                                    List<Runnable> pendingTasks = syncExecutor.shutdownNow();
311                                    LOG.info("pending tasks on stop {}", pendingTasks);
312                                }
313                                localBroker.oneway(new ShutdownInfo());
314                                remoteBroker.oneway(new ShutdownInfo());
315                            } catch (Throwable e) {
316                                LOG.debug("Caught exception sending shutdown", e);
317                            } finally {
318                                sendShutdown.countDown();
319                            }
320
321                        }
322                    }, "ActiveMQ ForwardingBridge StopTask");
323
324                    if (!sendShutdown.await(10, TimeUnit.SECONDS)) {
325                        LOG.info("Network Could not shutdown in a timely manner");
326                    }
327                } finally {
328                    ServiceStopper ss = new ServiceStopper();
329                    stopFailoverTransport(remoteBroker);
330                    ss.stop(remoteBroker);
331                    ss.stop(localBroker);
332                    ss.stop(duplexInboundLocalBroker);
333                    // Release the started Latch since another thread could be
334                    // stuck waiting for it to start up.
335                    startedLatch.countDown();
336                    startedLatch.countDown();
337                    localStartedLatch.countDown();
338                    staticDestinationsLatch.countDown();
339
340                    ss.throwFirstException();
341                }
342            }
343
344            LOG.info("{} bridge to {} stopped", configuration.getBrokerName(), remoteBrokerName);
345        }
346    }
347
348    private void stopFailoverTransport(Transport transport) {
349        FailoverTransport failoverTransport = transport.narrow(FailoverTransport.class);
350        if (failoverTransport != null) {
351            // may be blocked on write, in which case stop will block
352            try {
353                failoverTransport.handleTransportFailure(new IOException("Bridge stopped"));
354            } catch (InterruptedException ignored) {}
355        }
356    }
357
358    protected void triggerStartAsyncNetworkBridgeCreation() throws IOException {
359        brokerService.getTaskRunnerFactory().execute(new Runnable() {
360            @Override
361            public void run() {
362                final String originalName = Thread.currentThread().getName();
363                Thread.currentThread().setName("triggerStartAsyncNetworkBridgeCreation: " +
364                        "remoteBroker=" + remoteBroker + ", localBroker= " + localBroker);
365
366                try {
367                    // First we collect the info data from both the local and remote ends
368                    collectBrokerInfos();
369
370                    // Once we have all required broker info we can attempt to start
371                    // the local and then remote sides of the bridge.
372                    doStartLocalAndRemoteBridges();
373                } finally {
374                    Thread.currentThread().setName(originalName);
375                }
376            }
377        });
378    }
379
380    private void collectBrokerInfos() {
381        int timeout = 30000;
382        TcpTransport tcpTransport = remoteBroker.narrow(TcpTransport.class);
383        if (tcpTransport != null) {
384           timeout = tcpTransport.getConnectionTimeout();
385        }
386
387        // First wait for the remote to feed us its BrokerInfo, then we can check on
388        // the LocalBrokerInfo and decide is this is a loop.
389        try {
390            remoteBrokerInfo = futureRemoteBrokerInfo.get(timeout, TimeUnit.MILLISECONDS);
391            if (remoteBrokerInfo == null) {
392                serviceLocalException(new Throwable("remoteBrokerInfo is null"));
393                return;
394            }
395        } catch (Exception e) {
396            serviceRemoteException(e);
397            return;
398        }
399
400        try {
401            localBrokerInfo = futureLocalBrokerInfo.get(timeout, TimeUnit.MILLISECONDS);
402            if (localBrokerInfo == null) {
403                serviceLocalException(new Throwable("localBrokerInfo is null"));
404                return;
405            }
406
407            // Before we try and build the bridge lets check if we are in a loop
408            // and if so just stop now before registering anything.
409            remoteBrokerId = remoteBrokerInfo.getBrokerId();
410            if (localBrokerId.equals(remoteBrokerId)) {
411                LOG.trace("{} disconnecting remote loop back connector for: {}, with id: {}", new Object[]{
412                        configuration.getBrokerName(), remoteBrokerName, remoteBrokerId
413                });
414                ServiceSupport.dispose(localBroker);
415                ServiceSupport.dispose(remoteBroker);
416                // the bridge is left in a bit of limbo, but it won't get retried
417                // in this state.
418                return;
419            }
420
421            // Fill in the remote broker's information now.
422            remoteBrokerPath[0] = remoteBrokerId;
423            remoteBrokerName = remoteBrokerInfo.getBrokerName();
424            if (configuration.isUseBrokerNamesAsIdSeed()) {
425                idGenerator = new IdGenerator(brokerService.getBrokerName() + "->" + remoteBrokerName);
426            }
427        } catch (Throwable e) {
428            serviceLocalException(e);
429        }
430    }
431
432    private void doStartLocalAndRemoteBridges() {
433
434        if (disposed.get()) {
435            return;
436        }
437
438        if (isCreatedByDuplex()) {
439            // apply remote (propagated) configuration to local duplex bridge before start
440            Properties props = null;
441            try {
442                props = MarshallingSupport.stringToProperties(remoteBrokerInfo.getNetworkProperties());
443                IntrospectionSupport.getProperties(configuration, props, null);
444                if (configuration.getExcludedDestinations() != null) {
445                    excludedDestinations = configuration.getExcludedDestinations().toArray(
446                            new ActiveMQDestination[configuration.getExcludedDestinations().size()]);
447                }
448                if (configuration.getStaticallyIncludedDestinations() != null) {
449                    staticallyIncludedDestinations = configuration.getStaticallyIncludedDestinations().toArray(
450                            new ActiveMQDestination[configuration.getStaticallyIncludedDestinations().size()]);
451                }
452                if (configuration.getDynamicallyIncludedDestinations() != null) {
453                    dynamicallyIncludedDestinations = configuration.getDynamicallyIncludedDestinations().toArray(
454                            new ActiveMQDestination[configuration.getDynamicallyIncludedDestinations().size()]);
455                }
456            } catch (Throwable t) {
457                LOG.error("Error mapping remote configuration: {}", props, t);
458            }
459        }
460
461        try {
462            startLocalBridge();
463        } catch (Throwable e) {
464            serviceLocalException(e);
465            return;
466        }
467
468        try {
469            startRemoteBridge();
470        } catch (Throwable e) {
471            serviceRemoteException(e);
472            return;
473        }
474
475        try {
476            if (safeWaitUntilStarted()) {
477                setupStaticDestinations();
478                staticDestinationsLatch.countDown();
479            }
480        } catch (Throwable e) {
481            serviceLocalException(e);
482        }
483    }
484
485    private void startLocalBridge() throws Throwable {
486        if (!bridgeFailed.get() && localBridgeStarted.compareAndSet(false, true)) {
487            synchronized (this) {
488                LOG.trace("{} starting local Bridge, localBroker={}", configuration.getBrokerName(), localBroker);
489                if (!disposed.get()) {
490
491                    if (idGenerator == null) {
492                        throw new IllegalStateException("Id Generator cannot be null");
493                    }
494
495                    localConnectionInfo = new ConnectionInfo();
496                    localConnectionInfo.setConnectionId(new ConnectionId(idGenerator.generateId()));
497                    localClientId = configuration.getName() + configuration.getClientIdToken() + remoteBrokerName + configuration.getClientIdToken() + "inbound" + configuration.getClientIdToken() + configuration.getBrokerName();
498                    localConnectionInfo.setClientId(localClientId);
499                    localConnectionInfo.setUserName(configuration.getUserName());
500                    localConnectionInfo.setPassword(configuration.getPassword());
501                    Transport originalTransport = remoteBroker;
502                    while (originalTransport instanceof TransportFilter) {
503                        originalTransport = ((TransportFilter) originalTransport).getNext();
504                    }
505                    if (originalTransport instanceof TcpTransport) {
506                        X509Certificate[] peerCerts = originalTransport.getPeerCertificates();
507                        localConnectionInfo.setTransportContext(peerCerts);
508                    }
509                    // sync requests that may fail
510                    Object resp = localBroker.request(localConnectionInfo);
511                    if (resp instanceof ExceptionResponse) {
512                        throw ((ExceptionResponse) resp).getException();
513                    }
514                    localSessionInfo = new SessionInfo(localConnectionInfo, 1);
515                    localBroker.oneway(localSessionInfo);
516
517                    if (configuration.isDuplex()) {
518                        // separate in-bound channel for forwards so we don't
519                        // contend with out-bound dispatch on same connection
520                        remoteBrokerInfo.setNetworkConnection(true);
521                        duplexInboundLocalBroker.oneway(remoteBrokerInfo);
522
523                        ConnectionInfo duplexLocalConnectionInfo = new ConnectionInfo();
524                        duplexLocalConnectionInfo.setConnectionId(new ConnectionId(idGenerator.generateId()));
525                        duplexLocalConnectionInfo.setClientId(configuration.getName() + configuration.getClientIdToken() + remoteBrokerName + configuration.getClientIdToken() + "inbound" + configuration.getClientIdToken() + "duplex"
526                                + configuration.getClientIdToken() + configuration.getBrokerName());
527                        duplexLocalConnectionInfo.setUserName(configuration.getUserName());
528                        duplexLocalConnectionInfo.setPassword(configuration.getPassword());
529
530                        if (originalTransport instanceof TcpTransport) {
531                            X509Certificate[] peerCerts = originalTransport.getPeerCertificates();
532                            duplexLocalConnectionInfo.setTransportContext(peerCerts);
533                        }
534                        // sync requests that may fail
535                        resp = duplexInboundLocalBroker.request(duplexLocalConnectionInfo);
536                        if (resp instanceof ExceptionResponse) {
537                            throw ((ExceptionResponse) resp).getException();
538                        }
539                        SessionInfo duplexInboundSession = new SessionInfo(duplexLocalConnectionInfo, 1);
540                        duplexInboundLocalProducerInfo = new ProducerInfo(duplexInboundSession, 1);
541                        duplexInboundLocalBroker.oneway(duplexInboundSession);
542                        duplexInboundLocalBroker.oneway(duplexInboundLocalProducerInfo);
543                    }
544                    brokerService.getBroker().networkBridgeStarted(remoteBrokerInfo, this.createdByDuplex, remoteBroker.toString());
545                    NetworkBridgeListener l = this.networkBridgeListener;
546                    if (l != null) {
547                        l.onStart(this);
548                    }
549
550                    // Let the local broker know the remote broker's ID.
551                    localBroker.oneway(remoteBrokerInfo);
552                    // new peer broker (a consumer can work with remote broker also)
553                    brokerService.getBroker().addBroker(null, remoteBrokerInfo);
554
555                    LOG.info("Network connection between {} and {} ({}) has been established.", new Object[]{
556                            localBroker, remoteBroker, remoteBrokerName
557                    });
558                    LOG.trace("{} register bridge ({}) to {}", new Object[]{
559                            configuration.getBrokerName(), this, remoteBrokerName
560                    });
561                } else {
562                    LOG.warn("Bridge was disposed before the startLocalBridge() method was fully executed.");
563                }
564                startedLatch.countDown();
565                localStartedLatch.countDown();
566            }
567        }
568    }
569
570    protected void startRemoteBridge() throws Exception {
571        if (!bridgeFailed.get() && remoteBridgeStarted.compareAndSet(false, true)) {
572            LOG.trace("{} starting remote Bridge, remoteBroker={}", configuration.getBrokerName(), remoteBroker);
573            synchronized (this) {
574                if (!isCreatedByDuplex()) {
575                    BrokerInfo brokerInfo = new BrokerInfo();
576                    brokerInfo.setBrokerName(configuration.getBrokerName());
577                    brokerInfo.setBrokerURL(configuration.getBrokerURL());
578                    brokerInfo.setNetworkConnection(true);
579                    brokerInfo.setDuplexConnection(configuration.isDuplex());
580                    // set our properties
581                    Properties props = new Properties();
582                    IntrospectionSupport.getProperties(configuration, props, null);
583
584                    String dynamicallyIncludedDestinationsKey = "dynamicallyIncludedDestinations";
585                    String staticallyIncludedDestinationsKey = "staticallyIncludedDestinations";
586
587                    if (!configuration.getDynamicallyIncludedDestinations().isEmpty()) {
588                        props.put(dynamicallyIncludedDestinationsKey,
589                                StringToListOfActiveMQDestinationConverter.
590                                convertFromActiveMQDestination(configuration.getDynamicallyIncludedDestinations(), true));
591                    }
592                    if (!configuration.getStaticallyIncludedDestinations().isEmpty()) {
593                        props.put(staticallyIncludedDestinationsKey,
594                                StringToListOfActiveMQDestinationConverter.
595                                convertFromActiveMQDestination(configuration.getStaticallyIncludedDestinations(), true));
596                    }
597
598                    props.remove("networkTTL");
599                    String str = MarshallingSupport.propertiesToString(props);
600                    brokerInfo.setNetworkProperties(str);
601                    brokerInfo.setBrokerId(this.localBrokerId);
602                    remoteBroker.oneway(brokerInfo);
603                    if (configuration.isSyncDurableSubs() &&
604                            remoteBroker.getWireFormat().getVersion() >= CommandTypes.PROTOCOL_VERSION_DURABLE_SYNC) {
605                        remoteBroker.oneway(NetworkBridgeUtils.getBrokerSubscriptionInfo(brokerService,
606                                configuration));
607                    }
608                }
609                if (remoteConnectionInfo != null) {
610                    remoteBroker.oneway(remoteConnectionInfo.createRemoveCommand());
611                }
612                remoteConnectionInfo = new ConnectionInfo();
613                remoteConnectionInfo.setConnectionId(new ConnectionId(idGenerator.generateId()));
614                remoteConnectionInfo.setClientId(configuration.getName() + configuration.getClientIdToken() + configuration.getBrokerName() + configuration.getClientIdToken() + "outbound");
615                remoteConnectionInfo.setUserName(configuration.getUserName());
616                remoteConnectionInfo.setPassword(configuration.getPassword());
617                remoteBroker.oneway(remoteConnectionInfo);
618
619                SessionInfo remoteSessionInfo = new SessionInfo(remoteConnectionInfo, 1);
620                remoteBroker.oneway(remoteSessionInfo);
621                producerInfo = new ProducerInfo(remoteSessionInfo, 1);
622                producerInfo.setResponseRequired(false);
623                remoteBroker.oneway(producerInfo);
624                // Listen to consumer advisory messages on the remote broker to determine demand.
625                if (!configuration.isStaticBridge()) {
626                    demandConsumerInfo = new ConsumerInfo(remoteSessionInfo, 1);
627                    // always dispatch advisory message asynchronously so that
628                    // we never block the producer broker if we are slow
629                    demandConsumerInfo.setDispatchAsync(true);
630                    String advisoryTopic = configuration.getDestinationFilter();
631                    if (configuration.isBridgeTempDestinations()) {
632                        advisoryTopic += "," + AdvisorySupport.TEMP_DESTINATION_COMPOSITE_ADVISORY_TOPIC;
633                    }
634                    demandConsumerInfo.setDestination(new ActiveMQTopic(advisoryTopic));
635                    configureConsumerPrefetch(demandConsumerInfo);
636                    remoteBroker.oneway(demandConsumerInfo);
637                }
638                startedLatch.countDown();
639            }
640        }
641    }
642
643    @Override
644    public void serviceRemoteException(Throwable error) {
645        if (!disposed.get()) {
646            if (error instanceof SecurityException || error instanceof GeneralSecurityException) {
647                LOG.error("Network connection between {} and {} shutdown due to a remote error: {}", localBroker, remoteBroker, error.toString());
648            } else {
649                LOG.warn("Network connection between {} and {} shutdown due to a remote error: {}", localBroker, remoteBroker, error.toString());
650            }
651            LOG.debug("The remote Exception was: {}", error, error);
652            brokerService.getTaskRunnerFactory().execute(new Runnable() {
653                @Override
654                public void run() {
655                    ServiceSupport.dispose(getControllingService());
656                }
657            });
658            fireBridgeFailed(error);
659        }
660    }
661
662    /**
663     * Checks whether or not this consumer is a direct bridge network subscription
664     * @param info
665     * @return
666     */
667    protected boolean isDirectBridgeConsumer(ConsumerInfo info) {
668        return (info.getSubscriptionName() != null && info.getSubscriptionName().startsWith(DURABLE_SUB_PREFIX)) &&
669                (info.getClientId() == null || info.getClientId().startsWith(configuration.getName()));
670    }
671
672    protected boolean isProxyBridgeSubscription(String clientId, String subName) {
673        if (subName != null && clientId != null) {
674            if (subName.startsWith(DURABLE_SUB_PREFIX) && !clientId.startsWith(configuration.getName())) {
675                return true;
676            }
677        }
678        return false;
679    }
680
681    /**
682     * This scenaior is primarily used for durable sync on broker restarts
683     *
684     * @param sub
685     * @param clientId
686     * @param subName
687     */
688    protected void addProxyNetworkSubscriptionClientId(final DemandSubscription sub, final String clientId, String subName) {
689        if (clientId != null && sub != null && subName != null) {
690                String newClientId = getProxyBridgeClientId(clientId);
691                final SubscriptionInfo newSubInfo = new SubscriptionInfo(newClientId, subName);
692                sub.getDurableRemoteSubs().add(newSubInfo);
693                LOG.debug("Adding proxy network subscription {} to demand subscription", newSubInfo);
694
695        } else {
696            LOG.debug("Skipping addProxyNetworkSubscription");
697        }
698    }
699
700    /**
701     * Add a durable remote proxy subscription when we can generate via the BrokerId path
702     * This is the most common scenario
703     *
704     * @param sub
705     * @param path
706     * @param subName
707     */
708    protected void addProxyNetworkSubscriptionBrokerPath(final DemandSubscription sub, final BrokerId[] path, String subName) {
709        if (sub != null && path.length > 1 && subName != null) {
710            String b1 = path[path.length-1].toString();
711            String b2 = path[path.length-2].toString();
712            final SubscriptionInfo newSubInfo = new SubscriptionInfo(b2 + configuration.getClientIdToken() + "inbound" + configuration.getClientIdToken() + b1, subName);
713            sub.getDurableRemoteSubs().add(newSubInfo);
714        }
715    }
716
717    private String getProxyBridgeClientId(String clientId) {
718        String newClientId = clientId;
719        String[] clientIdTokens = newClientId != null ? newClientId.split(Pattern.quote(configuration.getClientIdToken())) : null;
720        if (clientIdTokens != null && clientIdTokens.length > 2) {
721            newClientId = clientIdTokens[clientIdTokens.length - 3] +  configuration.getClientIdToken() + "inbound"
722                    + configuration.getClientIdToken() +  clientIdTokens[clientIdTokens.length -1];
723        }
724        return newClientId;
725    }
726
727    protected boolean isProxyNSConsumerBrokerPath(ConsumerInfo info) {
728        return info.getBrokerPath() != null && info.getBrokerPath().length > 1;
729    }
730
731    protected boolean isProxyNSConsumerClientId(String clientId) {
732        return clientId != null && clientId.split(Pattern.quote(configuration.getClientIdToken())).length > 3;
733    }
734
735    protected void serviceRemoteCommand(Command command) {
736        if (!disposed.get()) {
737            try {
738                if (command.isMessageDispatch()) {
739                    safeWaitUntilStarted();
740                    MessageDispatch md = (MessageDispatch) command;
741                    serviceRemoteConsumerAdvisory(md.getMessage().getDataStructure());
742                    ackAdvisory(md.getMessage());
743                } else if (command.isBrokerInfo()) {
744                    futureRemoteBrokerInfo.set((BrokerInfo) command);
745                } else if (command instanceof BrokerSubscriptionInfo) {
746                    final BrokerSubscriptionInfo brokerSubscriptionInfo = (BrokerSubscriptionInfo) command;
747
748                    //Start in a new thread so we don't block the transport waiting for staticDestinations
749                    syncExecutor.execute(new Runnable() {
750
751                        @Override
752                        public void run() {
753                            try {
754                                staticDestinationsLatch.await();
755                                //Make sure after the countDown of staticDestinationsLatch we aren't stopping
756                                if (!disposed.get()) {
757                                    BrokerSubscriptionInfo subInfo = brokerSubscriptionInfo;
758                                    LOG.debug("Received Remote BrokerSubscriptionInfo on {} from {}",
759                                            brokerService.getBrokerName(), subInfo.getBrokerName());
760
761                                    if (configuration.isSyncDurableSubs() && configuration.isConduitSubscriptions()
762                                            && !configuration.isDynamicOnly()) {
763                                        if (started.get()) {
764                                            if (subInfo.getSubscriptionInfos() != null) {
765                                                for (ConsumerInfo info : subInfo.getSubscriptionInfos()) {
766                                                    //re-add any process any non-NC consumers that match the
767                                                    //dynamicallyIncludedDestinations list
768                                                    //Also re-add network consumers that are not part of this direct
769                                                    //bridge (proxy of proxy bridges)
770                                                    if((info.getSubscriptionName() == null || !isDirectBridgeConsumer(info)) &&
771                                                            NetworkBridgeUtils.matchesDestinations(dynamicallyIncludedDestinations, info.getDestination())) {
772                                                        serviceRemoteConsumerAdvisory(info);
773                                                    }
774                                                }
775                                            }
776
777                                            //After re-added, clean up any empty durables
778                                            for (Iterator<DemandSubscription> i = subscriptionMapByLocalId.values().iterator(); i.hasNext(); ) {
779                                                DemandSubscription ds = i.next();
780                                                if (NetworkBridgeUtils.matchesDestinations(dynamicallyIncludedDestinations, ds.getLocalInfo().getDestination())) {
781                                                    cleanupDurableSub(ds, i);
782                                                }
783                                            }
784                                        }
785                                    }
786                                }
787                            } catch (Exception e) {
788                                LOG.warn("Error processing BrokerSubscriptionInfo: {}", e.getMessage(), e);
789                                LOG.debug(e.getMessage(), e);
790                            }
791                        }
792                    });
793
794                } else if (command.getClass() == ConnectionError.class) {
795                    ConnectionError ce = (ConnectionError) command;
796                    serviceRemoteException(ce.getException());
797                } else {
798                    if (isDuplex()) {
799                        LOG.trace("{} duplex command type: {}", configuration.getBrokerName(), command.getDataStructureType());
800                        if (command.isMessage()) {
801                            final ActiveMQMessage message = (ActiveMQMessage) command;
802                            if (NetworkBridgeFilter.isAdvisoryInterpretedByNetworkBridge(message)) {
803                                serviceRemoteConsumerAdvisory(message.getDataStructure());
804                                ackAdvisory(message);
805                            } else {
806                                if (!isPermissableDestination(message.getDestination(), true)) {
807                                    return;
808                                }
809                                safeWaitUntilStarted();
810                                // message being forwarded - we need to
811                                // propagate the response to our local send
812                                if (canDuplexDispatch(message)) {
813                                    message.setProducerId(duplexInboundLocalProducerInfo.getProducerId());
814                                    if (message.isResponseRequired() || configuration.isAlwaysSyncSend()) {
815                                        duplexInboundLocalBroker.asyncRequest(message, new ResponseCallback() {
816                                            final int correlationId = message.getCommandId();
817
818                                            @Override
819                                            public void onCompletion(FutureResponse resp) {
820                                                try {
821                                                    Response reply = resp.getResult();
822                                                    reply.setCorrelationId(correlationId);
823                                                    remoteBroker.oneway(reply);
824                                                    //increment counter when messages are received in duplex mode
825                                                    networkBridgeStatistics.getReceivedCount().increment();
826                                                } catch (IOException error) {
827                                                    LOG.error("Exception: {} on duplex forward of: {}", error, message);
828                                                    serviceRemoteException(error);
829                                                }
830                                            }
831                                        });
832                                    } else {
833                                        duplexInboundLocalBroker.oneway(message);
834                                        networkBridgeStatistics.getReceivedCount().increment();
835                                    }
836                                    serviceInboundMessage(message);
837                                } else {
838                                    if (message.isResponseRequired() || configuration.isAlwaysSyncSend()) {
839                                        Response reply = new Response();
840                                        reply.setCorrelationId(message.getCommandId());
841                                        remoteBroker.oneway(reply);
842                                    }
843                                }
844                            }
845                        } else {
846                            switch (command.getDataStructureType()) {
847                                case ConnectionInfo.DATA_STRUCTURE_TYPE:
848                                    if (duplexInitiatingConnection != null && duplexInitiatingConnectionInfoReceived.compareAndSet(false, true)) {
849                                        // end of initiating connection setup - propogate to initial connection to get mbean by clientid
850                                        duplexInitiatingConnection.processAddConnection((ConnectionInfo) command);
851                                    } else {
852                                        localBroker.oneway(command);
853                                    }
854                                    break;
855                                case SessionInfo.DATA_STRUCTURE_TYPE:
856                                    localBroker.oneway(command);
857                                    break;
858                                case ProducerInfo.DATA_STRUCTURE_TYPE:
859                                    // using duplexInboundLocalProducerInfo
860                                    break;
861                                case MessageAck.DATA_STRUCTURE_TYPE:
862                                    MessageAck ack = (MessageAck) command;
863                                    DemandSubscription localSub = subscriptionMapByRemoteId.get(ack.getConsumerId());
864                                    if (localSub != null) {
865                                        ack.setConsumerId(localSub.getLocalInfo().getConsumerId());
866                                        localBroker.oneway(ack);
867                                    } else {
868                                        LOG.warn("Matching local subscription not found for ack: {}", ack);
869                                    }
870                                    break;
871                                case ConsumerInfo.DATA_STRUCTURE_TYPE:
872                                    localStartedLatch.await();
873                                    if (started.get()) {
874                                        final ConsumerInfo consumerInfo = (ConsumerInfo) command;
875                                        if (isDuplicateSuppressionOff(consumerInfo)) {
876                                            addConsumerInfo(consumerInfo);
877                                        } else {
878                                            synchronized (brokerService.getVmConnectorURI()) {
879                                                addConsumerInfo(consumerInfo);
880                                            }
881                                        }
882                                    } else {
883                                        // received a subscription whilst stopping
884                                        LOG.warn("Stopping - ignoring ConsumerInfo: {}", command);
885                                    }
886                                    break;
887                                case ShutdownInfo.DATA_STRUCTURE_TYPE:
888                                    // initiator is shutting down, controlled case
889                                    // abortive close dealt with by inactivity monitor
890                                    LOG.info("Stopping network bridge on shutdown of remote broker");
891                                    serviceRemoteException(new IOException(command.toString()));
892                                    break;
893                                default:
894                                    LOG.debug("Ignoring remote command: {}", command);
895                            }
896                        }
897                    } else {
898                        switch (command.getDataStructureType()) {
899                            case KeepAliveInfo.DATA_STRUCTURE_TYPE:
900                            case WireFormatInfo.DATA_STRUCTURE_TYPE:
901                            case ShutdownInfo.DATA_STRUCTURE_TYPE:
902                                break;
903                            default:
904                                LOG.warn("Unexpected remote command: {}", command);
905                        }
906                    }
907                }
908            } catch (Throwable e) {
909                LOG.debug("Exception processing remote command: {}", command, e);
910                serviceRemoteException(e);
911            }
912        }
913    }
914
915    private void ackAdvisory(Message message) throws IOException {
916        demandConsumerDispatched++;
917        if (demandConsumerDispatched > (demandConsumerInfo.getPrefetchSize() *
918                (configuration.getAdvisoryAckPercentage() / 100f))) {
919            final MessageAck ack = new MessageAck(message, MessageAck.STANDARD_ACK_TYPE, demandConsumerDispatched);
920            ack.setConsumerId(demandConsumerInfo.getConsumerId());
921            brokerService.getTaskRunnerFactory().execute(new Runnable() {
922                @Override
923                public void run() {
924                    try {
925                        remoteBroker.oneway(ack);
926                    } catch (IOException e) {
927                        LOG.warn("Failed to send advisory ack " + ack, e);
928                    }
929                }
930            });
931            demandConsumerDispatched = 0;
932        }
933    }
934
935    private void serviceRemoteConsumerAdvisory(DataStructure data) throws IOException {
936        final int networkTTL = configuration.getConsumerTTL();
937        if (data.getClass() == ConsumerInfo.class) {
938            // Create a new local subscription
939            ConsumerInfo info = (ConsumerInfo) data;
940            BrokerId[] path = info.getBrokerPath();
941
942            if (info.isBrowser()) {
943                LOG.debug("{} Ignoring sub from {}, browsers explicitly suppressed", configuration.getBrokerName(), remoteBrokerName);
944                return;
945            }
946
947            if (path != null && networkTTL > -1 && path.length >= networkTTL) {
948                LOG.debug("{} Ignoring sub from {}, restricted to {} network hops only: {}", new Object[]{
949                        configuration.getBrokerName(), remoteBrokerName, networkTTL, info
950                });
951                return;
952            }
953
954            if (contains(path, localBrokerPath[0])) {
955                // Ignore this consumer as it's a consumer we locally sent to the broker.
956                LOG.debug("{} Ignoring sub from {}, already routed through this broker once: {}", new Object[]{
957                        configuration.getBrokerName(), remoteBrokerName, info
958                });
959                return;
960            }
961
962            if (!isPermissableDestination(info.getDestination())) {
963                // ignore if not in the permitted or in the excluded list
964                LOG.debug("{} Ignoring sub from {}, destination {} is not permitted: {}", new Object[]{
965                        configuration.getBrokerName(), remoteBrokerName, info.getDestination(), info
966                });
967                return;
968            }
969
970            // in a cyclic network there can be multiple bridges per broker that can propagate
971            // a network subscription so there is a need to synchronize on a shared entity
972            // if duplicate suppression is required
973            if (isDuplicateSuppressionOff(info)) {
974                addConsumerInfo(info);
975            } else {
976                synchronized (brokerService.getVmConnectorURI()) {
977                    addConsumerInfo(info);
978                }
979            }
980        } else if (data.getClass() == DestinationInfo.class) {
981            // It's a destination info - we want to pass up information about temporary destinations
982            final DestinationInfo destInfo = (DestinationInfo) data;
983            BrokerId[] path = destInfo.getBrokerPath();
984            if (path != null && networkTTL > -1 && path.length >= networkTTL) {
985                LOG.debug("{} Ignoring destination {} restricted to {} network hops only", new Object[]{
986                        configuration.getBrokerName(), destInfo, networkTTL
987                });
988                return;
989            }
990            if (contains(destInfo.getBrokerPath(), localBrokerPath[0])) {
991                LOG.debug("{} Ignoring destination {} already routed through this broker once", configuration.getBrokerName(), destInfo);
992                return;
993            }
994            destInfo.setConnectionId(localConnectionInfo.getConnectionId());
995            if (destInfo.getDestination() instanceof ActiveMQTempDestination) {
996                // re-set connection id so comes from here
997                ActiveMQTempDestination tempDest = (ActiveMQTempDestination) destInfo.getDestination();
998                tempDest.setConnectionId(localSessionInfo.getSessionId().getConnectionId());
999            }
1000            destInfo.setBrokerPath(appendToBrokerPath(destInfo.getBrokerPath(), getRemoteBrokerPath()));
1001            LOG.trace("{} bridging {} destination on {} from {}, destination: {}", new Object[]{
1002                    configuration.getBrokerName(), (destInfo.isAddOperation() ? "add" : "remove"), localBroker, remoteBrokerName, destInfo
1003            });
1004            if (destInfo.isRemoveOperation()) {
1005                // Serialize with removeSub operations such that all removeSub advisories
1006                // are generated
1007                serialExecutor.execute(new Runnable() {
1008                    @Override
1009                    public void run() {
1010                        try {
1011                            localBroker.oneway(destInfo);
1012                        } catch (IOException e) {
1013                            LOG.warn("failed to deliver remove command for destination: {}", destInfo.getDestination(), e);
1014                        }
1015                    }
1016                });
1017            } else {
1018                localBroker.oneway(destInfo);
1019            }
1020        } else if (data.getClass() == RemoveInfo.class) {
1021            ConsumerId id = (ConsumerId) ((RemoveInfo) data).getObjectId();
1022            removeDemandSubscription(id);
1023
1024            if (forcedDurableRemoteId.remove(id)) {
1025                for (Iterator<DemandSubscription> i = subscriptionMapByLocalId.values().iterator(); i.hasNext(); ) {
1026                    DemandSubscription ds = i.next();
1027                    boolean removed = ds.removeForcedDurableConsumer(id);
1028                    if (removed) {
1029                        cleanupDurableSub(ds, i);
1030                    }
1031                }
1032           }
1033
1034        } else if (data.getClass() == RemoveSubscriptionInfo.class) {
1035            final RemoveSubscriptionInfo info = ((RemoveSubscriptionInfo) data);
1036            final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(info.getClientId(), info.getSubscriptionName());
1037            final boolean proxyBridgeSub = isProxyBridgeSubscription(subscriptionInfo.getClientId(),
1038                    subscriptionInfo.getSubscriptionName());
1039            for (Iterator<DemandSubscription> i = subscriptionMapByLocalId.values().iterator(); i.hasNext(); ) {
1040                DemandSubscription ds = i.next();
1041                boolean removed = ds.getDurableRemoteSubs().remove(subscriptionInfo);
1042
1043                //If this is a proxy bridge subscription we need to try changing the clientId
1044                if (!removed && proxyBridgeSub){
1045                    subscriptionInfo.setClientId(getProxyBridgeClientId(subscriptionInfo.getClientId()));
1046                    if (ds.getDurableRemoteSubs().contains(subscriptionInfo)) {
1047                        ds.getDurableRemoteSubs().remove(subscriptionInfo);
1048                        removed = true;
1049                    }
1050                }
1051
1052                if (removed) {
1053                    cleanupDurableSub(ds, i);
1054                }
1055            }
1056        }
1057    }
1058
1059    private void cleanupDurableSub(final DemandSubscription ds,
1060            Iterator<DemandSubscription> i) throws IOException {
1061
1062        if (ds != null && ds.getLocalDurableSubscriber() != null && ds.getDurableRemoteSubs().isEmpty()
1063                && ds.getForcedDurableConsumersSize() == 0) {
1064            // deactivate subscriber
1065            RemoveInfo removeInfo = new RemoveInfo(ds.getLocalInfo().getConsumerId());
1066            localBroker.oneway(removeInfo);
1067
1068            // remove subscriber
1069            RemoveSubscriptionInfo sending = new RemoveSubscriptionInfo();
1070            sending.setClientId(localClientId);
1071            sending.setSubscriptionName(ds.getLocalDurableSubscriber().getSubscriptionName());
1072            sending.setConnectionId(this.localConnectionInfo.getConnectionId());
1073            localBroker.oneway(sending);
1074
1075            //remove subscriber from local map
1076            i.remove();
1077
1078            //need to remove the mapping from the remote map as well
1079            subscriptionMapByRemoteId.remove(ds.getRemoteInfo().getConsumerId());
1080        }
1081    }
1082
1083    @Override
1084    public void serviceLocalException(Throwable error) {
1085        serviceLocalException(null, error);
1086    }
1087
1088    public void serviceLocalException(MessageDispatch messageDispatch, Throwable error) {
1089        LOG.trace("serviceLocalException: disposed {} ex", disposed.get(), error);
1090        if (!disposed.get()) {
1091            if (error instanceof DestinationDoesNotExistException && ((DestinationDoesNotExistException) error).isTemporary()) {
1092                // not a reason to terminate the bridge - temps can disappear with
1093                // pending sends as the demand sub may outlive the remote dest
1094                if (messageDispatch != null) {
1095                    LOG.warn("PoisonAck of {} on forwarding error: {}", messageDispatch.getMessage().getMessageId(), error);
1096                    try {
1097                        MessageAck poisonAck = new MessageAck(messageDispatch, MessageAck.POISON_ACK_TYPE, 1);
1098                        poisonAck.setPoisonCause(error);
1099                        localBroker.oneway(poisonAck);
1100                    } catch (IOException ioe) {
1101                        LOG.error("Failed to poison ack message following forward failure: ", ioe);
1102                    }
1103                    fireFailedForwardAdvisory(messageDispatch, error);
1104                } else {
1105                    LOG.warn("Ignoring exception on forwarding to non existent temp dest: ", error);
1106                }
1107                return;
1108            }
1109
1110            LOG.info("Network connection between {} and {} shutdown due to a local error: {}", new Object[]{localBroker, remoteBroker, error});
1111            LOG.debug("The local Exception was: {}", error, error);
1112
1113            brokerService.getTaskRunnerFactory().execute(new Runnable() {
1114                @Override
1115                public void run() {
1116                    ServiceSupport.dispose(getControllingService());
1117                }
1118            });
1119            fireBridgeFailed(error);
1120        }
1121    }
1122
1123    private void fireFailedForwardAdvisory(MessageDispatch messageDispatch, Throwable error) {
1124        if (configuration.isAdvisoryForFailedForward()) {
1125            AdvisoryBroker advisoryBroker = null;
1126            try {
1127                advisoryBroker = (AdvisoryBroker) brokerService.getBroker().getAdaptor(AdvisoryBroker.class);
1128
1129                if (advisoryBroker != null) {
1130                    ConnectionContext context = new ConnectionContext();
1131                    context.setSecurityContext(SecurityContext.BROKER_SECURITY_CONTEXT);
1132                    context.setBroker(brokerService.getBroker());
1133
1134                    ActiveMQMessage advisoryMessage = new ActiveMQMessage();
1135                    advisoryMessage.setStringProperty("cause", error.getLocalizedMessage());
1136                    advisoryBroker.fireAdvisory(context, AdvisorySupport.getNetworkBridgeForwardFailureAdvisoryTopic(), messageDispatch.getMessage(), null,
1137                            advisoryMessage);
1138
1139                }
1140            } catch (Exception e) {
1141                LOG.warn("failed to fire forward failure advisory, cause: {}", (Object)e);
1142                LOG.debug("detail", e);
1143            }
1144        }
1145    }
1146
1147    protected Service getControllingService() {
1148        return duplexInitiatingConnection != null ? duplexInitiatingConnection : DemandForwardingBridgeSupport.this;
1149    }
1150
1151    protected void addSubscription(DemandSubscription sub) throws IOException {
1152        if (sub != null) {
1153            localBroker.oneway(sub.getLocalInfo());
1154        }
1155    }
1156
1157    protected void removeSubscription(final DemandSubscription sub) throws IOException {
1158        if (sub != null) {
1159            LOG.trace("{} remove local subscription: {} for remote {}", new Object[]{configuration.getBrokerName(), sub.getLocalInfo().getConsumerId(), sub.getRemoteInfo().getConsumerId()});
1160
1161            // ensure not available for conduit subs pending removal
1162            subscriptionMapByLocalId.remove(sub.getLocalInfo().getConsumerId());
1163            subscriptionMapByRemoteId.remove(sub.getRemoteInfo().getConsumerId());
1164
1165            // continue removal in separate thread to free up tshis thread for outstanding responses
1166            // Serialize with removeDestination operations so that removeSubs are serialized with
1167            // removeDestinations such that all removeSub advisories are generated
1168            serialExecutor.execute(new Runnable() {
1169                @Override
1170                public void run() {
1171                    sub.waitForCompletion();
1172                    try {
1173                        localBroker.oneway(sub.getLocalInfo().createRemoveCommand());
1174                    } catch (IOException e) {
1175                        LOG.warn("failed to deliver remove command for local subscription, for remote {}", sub.getRemoteInfo().getConsumerId(), e);
1176                    }
1177                }
1178            });
1179        }
1180    }
1181
1182    protected Message configureMessage(MessageDispatch md) throws IOException {
1183        Message message = md.getMessage().copy();
1184        // Update the packet to show where it came from.
1185        message.setBrokerPath(appendToBrokerPath(message.getBrokerPath(), localBrokerPath));
1186        message.setProducerId(producerInfo.getProducerId());
1187        message.setDestination(md.getDestination());
1188        message.setMemoryUsage(null);
1189        if (message.getOriginalTransactionId() == null) {
1190            message.setOriginalTransactionId(message.getTransactionId());
1191        }
1192        message.setTransactionId(null);
1193        if (configuration.isUseCompression()) {
1194            message.compress();
1195        }
1196        return message;
1197    }
1198
1199    protected void serviceLocalCommand(Command command) {
1200        if (!disposed.get()) {
1201            try {
1202                if (command.isMessageDispatch()) {
1203                    safeWaitUntilStarted();
1204                    networkBridgeStatistics.getEnqueues().increment();
1205                    final MessageDispatch md = (MessageDispatch) command;
1206                    final DemandSubscription sub = subscriptionMapByLocalId.get(md.getConsumerId());
1207                    if (sub != null && md.getMessage() != null && sub.incrementOutstandingResponses()) {
1208
1209                        if (suppressMessageDispatch(md, sub)) {
1210                            LOG.debug("{} message not forwarded to {} because message came from there or fails TTL, brokerPath: {}, message: {}", new Object[]{
1211                                    configuration.getBrokerName(), remoteBrokerName, Arrays.toString(md.getMessage().getBrokerPath()), md.getMessage()
1212                            });
1213                            // still ack as it may be durable
1214                            try {
1215                                localBroker.oneway(new MessageAck(md, MessageAck.INDIVIDUAL_ACK_TYPE, 1));
1216                            } finally {
1217                                sub.decrementOutstandingResponses();
1218                            }
1219                            return;
1220                        }
1221
1222                        Message message = configureMessage(md);
1223                        LOG.debug("bridging ({} -> {}), consumer: {}, destination: {}, brokerPath: {}, message: {}", new Object[]{
1224                                configuration.getBrokerName(), remoteBrokerName, md.getConsumerId(), message.getDestination(), Arrays.toString(message.getBrokerPath()), (LOG.isTraceEnabled() ? message : message.getMessageId())
1225                        });
1226                        if (isDuplex() && NetworkBridgeFilter.isAdvisoryInterpretedByNetworkBridge(message)) {
1227                            try {
1228                                // never request b/c they are eventually                     acked async
1229                                remoteBroker.oneway(message);
1230                            } finally {
1231                                sub.decrementOutstandingResponses();
1232                            }
1233                            return;
1234                        }
1235                        if (isPermissableDestination(md.getDestination())) {
1236                           if (message.isPersistent() || configuration.isAlwaysSyncSend()) {
1237
1238                              // The message was not sent using async send, so we should only
1239                              // ack the local broker when we get confirmation that the remote
1240                              // broker has received the message.
1241                              remoteBroker.asyncRequest(message, new ResponseCallback() {
1242                                 @Override
1243                                 public void onCompletion(FutureResponse future) {
1244                                    try {
1245                                       Response response = future.getResult();
1246                                       if (response.isException()) {
1247                                          ExceptionResponse er = (ExceptionResponse) response;
1248                                          serviceLocalException(md, er.getException());
1249                                       } else {
1250                                          localBroker.oneway(new MessageAck(md, MessageAck.INDIVIDUAL_ACK_TYPE, 1));
1251                                          networkBridgeStatistics.getDequeues().increment();
1252                                       }
1253                                    } catch (IOException e) {
1254                                       serviceLocalException(md, e);
1255                                    } finally {
1256                                       sub.decrementOutstandingResponses();
1257                                    }
1258                                 }
1259                              });
1260
1261                           } else {
1262                              // If the message was originally sent using async send, we will
1263                              // preserve that QOS by bridging it using an async send (small chance
1264                              // of message loss).
1265                              try {
1266                                 remoteBroker.oneway(message);
1267                                 localBroker.oneway(new MessageAck(md, MessageAck.INDIVIDUAL_ACK_TYPE, 1));
1268                                 networkBridgeStatistics.getDequeues().increment();
1269                              } finally {
1270                                 sub.decrementOutstandingResponses();
1271                              }
1272                           }
1273                           serviceOutbound(message);
1274                        }
1275                    } else {
1276                        LOG.debug("No subscription registered with this network bridge for consumerId: {} for message: {}", md.getConsumerId(), md.getMessage());
1277                    }
1278                } else if (command.isBrokerInfo()) {
1279                    futureLocalBrokerInfo.set((BrokerInfo) command);
1280                } else if (command.isShutdownInfo()) {
1281                    LOG.info("{} Shutting down {}", configuration.getBrokerName(), configuration.getName());
1282                    stop();
1283                } else if (command.getClass() == ConnectionError.class) {
1284                    ConnectionError ce = (ConnectionError) command;
1285                    serviceLocalException(ce.getException());
1286                } else {
1287                    switch (command.getDataStructureType()) {
1288                        case WireFormatInfo.DATA_STRUCTURE_TYPE:
1289                            break;
1290                        case BrokerSubscriptionInfo.DATA_STRUCTURE_TYPE:
1291                            break;
1292                        default:
1293                            LOG.warn("Unexpected local command: {}", command);
1294                    }
1295                }
1296            } catch (Throwable e) {
1297                LOG.warn("Caught an exception processing local command", e);
1298                serviceLocalException(e);
1299            }
1300        }
1301    }
1302
1303    private boolean suppressMessageDispatch(MessageDispatch md, DemandSubscription sub) throws Exception {
1304        boolean suppress = false;
1305        // for durable subs, suppression via filter leaves dangling acks so we
1306        // need to check here and allow the ack irrespective
1307        if (sub.getLocalInfo().isDurable()) {
1308            NonCachedMessageEvaluationContext messageEvalContext = new NonCachedMessageEvaluationContext();
1309            messageEvalContext.setMessageReference(md.getMessage());
1310            messageEvalContext.setDestination(md.getDestination());
1311            suppress = !sub.getNetworkBridgeFilter().matches(messageEvalContext);
1312        }
1313        return suppress;
1314    }
1315
1316    public static boolean contains(BrokerId[] brokerPath, BrokerId brokerId) {
1317        if (brokerPath != null) {
1318            for (BrokerId id : brokerPath) {
1319                if (brokerId.equals(id)) {
1320                    return true;
1321                }
1322            }
1323        }
1324        return false;
1325    }
1326
1327    protected BrokerId[] appendToBrokerPath(BrokerId[] brokerPath, BrokerId[] pathsToAppend) {
1328        if (brokerPath == null || brokerPath.length == 0) {
1329            return pathsToAppend;
1330        }
1331        BrokerId rc[] = new BrokerId[brokerPath.length + pathsToAppend.length];
1332        System.arraycopy(brokerPath, 0, rc, 0, brokerPath.length);
1333        System.arraycopy(pathsToAppend, 0, rc, brokerPath.length, pathsToAppend.length);
1334        return rc;
1335    }
1336
1337    protected BrokerId[] appendToBrokerPath(BrokerId[] brokerPath, BrokerId idToAppend) {
1338        if (brokerPath == null || brokerPath.length == 0) {
1339            return new BrokerId[]{idToAppend};
1340        }
1341        BrokerId rc[] = new BrokerId[brokerPath.length + 1];
1342        System.arraycopy(brokerPath, 0, rc, 0, brokerPath.length);
1343        rc[brokerPath.length] = idToAppend;
1344        return rc;
1345    }
1346
1347    protected boolean isPermissableDestination(ActiveMQDestination destination) {
1348        return isPermissableDestination(destination, false);
1349    }
1350
1351    protected boolean isPermissableDestination(ActiveMQDestination destination, boolean allowTemporary) {
1352        // Are we not bridging temporary destinations?
1353        if (destination.isTemporary()) {
1354            if (allowTemporary) {
1355                return true;
1356            } else {
1357                return configuration.isBridgeTempDestinations();
1358            }
1359        }
1360
1361        ActiveMQDestination[] dests = excludedDestinations;
1362        if (dests != null && dests.length > 0) {
1363            for (ActiveMQDestination dest : dests) {
1364                DestinationFilter exclusionFilter = DestinationFilter.parseFilter(dest);
1365                if (dest != null && exclusionFilter.matches(destination) && dest.getDestinationType() == destination.getDestinationType()) {
1366                    return false;
1367                }
1368            }
1369        }
1370
1371        dests = staticallyIncludedDestinations;
1372        if (dests != null && dests.length > 0) {
1373            for (ActiveMQDestination dest : dests) {
1374                DestinationFilter inclusionFilter = DestinationFilter.parseFilter(dest);
1375                if (dest != null && inclusionFilter.matches(destination) && dest.getDestinationType() == destination.getDestinationType()) {
1376                    return true;
1377                }
1378            }
1379        }
1380
1381        dests = dynamicallyIncludedDestinations;
1382        if (dests != null && dests.length > 0) {
1383            for (ActiveMQDestination dest : dests) {
1384                DestinationFilter inclusionFilter = DestinationFilter.parseFilter(dest);
1385                if (dest != null && inclusionFilter.matches(destination) && dest.getDestinationType() == destination.getDestinationType()) {
1386                    return true;
1387                }
1388            }
1389
1390            return false;
1391        }
1392
1393        return true;
1394    }
1395
1396    /**
1397     * Subscriptions for these destinations are always created
1398     */
1399    protected void setupStaticDestinations() {
1400        ActiveMQDestination[] dests = staticallyIncludedDestinations;
1401        if (dests != null) {
1402            for (ActiveMQDestination dest : dests) {
1403                if (isPermissableDestination(dest)) {
1404                    DemandSubscription sub = createDemandSubscription(dest, null);
1405                    if (sub != null) {
1406                        sub.setStaticallyIncluded(true);
1407                        try {
1408                            addSubscription(sub);
1409                        } catch (IOException e) {
1410                            LOG.error("Failed to add static destination {}", dest, e);
1411                        }
1412                        LOG.trace("{}, bridging messages for static destination: {}", configuration.getBrokerName(), dest);
1413                    } else {
1414                        LOG.info("{}, static destination excluded: {}, demand already exists", configuration.getBrokerName(), dest);
1415                    }
1416                } else {
1417                    LOG.info("{}, static destination excluded: {}", configuration.getBrokerName(), dest);
1418                }
1419            }
1420        }
1421    }
1422
1423    protected void addConsumerInfo(final ConsumerInfo consumerInfo) throws IOException {
1424        ConsumerInfo info = consumerInfo.copy();
1425        addRemoteBrokerToBrokerPath(info);
1426        DemandSubscription sub = createDemandSubscription(info);
1427        if (sub != null) {
1428            if (duplicateSuppressionIsRequired(sub)) {
1429                undoMapRegistration(sub);
1430            } else {
1431                if (consumerInfo.isDurable()) {
1432                    //Handle the demand generated by proxy network subscriptions
1433                    //The broker path is case is normal
1434                    if (isProxyNSConsumerBrokerPath(sub.getRemoteInfo()) &&
1435                            info.getSubscriptionName() != null && info.getSubscriptionName().startsWith(DURABLE_SUB_PREFIX)) {
1436                        final BrokerId[] path = info.getBrokerPath();
1437                        addProxyNetworkSubscriptionBrokerPath(sub, path, consumerInfo.getSubscriptionName());
1438                    //This is the durable sync case on broker restart
1439                    } else if (isProxyNSConsumerClientId(sub.getRemoteInfo().getClientId()) &&
1440                            isProxyBridgeSubscription(info.getClientId(), info.getSubscriptionName())) {
1441                                addProxyNetworkSubscriptionClientId(sub, sub.getRemoteInfo().getClientId(), consumerInfo.getSubscriptionName());
1442                        } else {
1443                                sub.getDurableRemoteSubs().add(new SubscriptionInfo(sub.getRemoteInfo().getClientId(), consumerInfo.getSubscriptionName()));
1444                        }
1445                }
1446                addSubscription(sub);
1447                LOG.debug("{} new demand subscription: {}", configuration.getBrokerName(), sub);
1448            }
1449        }
1450    }
1451
1452    private void undoMapRegistration(DemandSubscription sub) {
1453        subscriptionMapByLocalId.remove(sub.getLocalInfo().getConsumerId());
1454        subscriptionMapByRemoteId.remove(sub.getRemoteInfo().getConsumerId());
1455    }
1456
1457    /*
1458     * check our existing subs networkConsumerIds against the list of network
1459     * ids in this subscription A match means a duplicate which we suppress for
1460     * topics and maybe for queues
1461     */
1462    private boolean duplicateSuppressionIsRequired(DemandSubscription candidate) {
1463        final ConsumerInfo consumerInfo = candidate.getRemoteInfo();
1464        boolean suppress = false;
1465
1466        if (isDuplicateSuppressionOff(consumerInfo)) {
1467            return suppress;
1468        }
1469
1470        List<ConsumerId> candidateConsumers = consumerInfo.getNetworkConsumerIds();
1471        Collection<Subscription> currentSubs = getRegionSubscriptions(consumerInfo.getDestination());
1472        for (Subscription sub : currentSubs) {
1473            List<ConsumerId> networkConsumers = sub.getConsumerInfo().getNetworkConsumerIds();
1474            if (!networkConsumers.isEmpty()) {
1475                if (matchFound(candidateConsumers, networkConsumers)) {
1476                    if (isInActiveDurableSub(sub)) {
1477                        suppress = false;
1478                    } else {
1479                        suppress = hasLowerPriority(sub, candidate.getLocalInfo());
1480                    }
1481                    break;
1482                }
1483            }
1484        }
1485        return suppress;
1486    }
1487
1488    private boolean isDuplicateSuppressionOff(final ConsumerInfo consumerInfo) {
1489        return !configuration.isSuppressDuplicateQueueSubscriptions() && !configuration.isSuppressDuplicateTopicSubscriptions()
1490                || consumerInfo.getDestination().isQueue() && !configuration.isSuppressDuplicateQueueSubscriptions()
1491                || consumerInfo.getDestination().isTopic() && !configuration.isSuppressDuplicateTopicSubscriptions();
1492    }
1493
1494    private boolean isInActiveDurableSub(Subscription sub) {
1495        return (sub.getConsumerInfo().isDurable() && sub instanceof DurableTopicSubscription && !((DurableTopicSubscription) sub).isActive());
1496    }
1497
1498    private boolean hasLowerPriority(Subscription existingSub, ConsumerInfo candidateInfo) {
1499        boolean suppress = false;
1500
1501        if (existingSub.getConsumerInfo().getPriority() >= candidateInfo.getPriority()) {
1502            LOG.debug("{} Ignoring duplicate subscription from {}, sub: {} is duplicate by network subscription with equal or higher network priority: {}, networkConsumerIds: {}", new Object[]{
1503                    configuration.getBrokerName(), remoteBrokerName, candidateInfo, existingSub, existingSub.getConsumerInfo().getNetworkConsumerIds()
1504            });
1505            suppress = true;
1506        } else {
1507            // remove the existing lower priority duplicate and allow this candidate
1508            try {
1509                removeDuplicateSubscription(existingSub);
1510
1511                LOG.debug("{} Replacing duplicate subscription {} with sub from {}, which has a higher priority, new sub: {}, networkConsumerIds: {}", new Object[]{
1512                        configuration.getBrokerName(), existingSub.getConsumerInfo(), remoteBrokerName, candidateInfo, candidateInfo.getNetworkConsumerIds()
1513                });
1514            } catch (IOException e) {
1515                LOG.error("Failed to remove duplicated sub as a result of sub with higher priority, sub: {}", existingSub, e);
1516            }
1517        }
1518        return suppress;
1519    }
1520
1521    private void removeDuplicateSubscription(Subscription existingSub) throws IOException {
1522        for (NetworkConnector connector : brokerService.getNetworkConnectors()) {
1523            if (connector.removeDemandSubscription(existingSub.getConsumerInfo().getConsumerId())) {
1524                break;
1525            }
1526        }
1527    }
1528
1529    private boolean matchFound(List<ConsumerId> candidateConsumers, List<ConsumerId> networkConsumers) {
1530        boolean found = false;
1531        for (ConsumerId aliasConsumer : networkConsumers) {
1532            if (candidateConsumers.contains(aliasConsumer)) {
1533                found = true;
1534                break;
1535            }
1536        }
1537        return found;
1538    }
1539
1540    protected final Collection<Subscription> getRegionSubscriptions(ActiveMQDestination dest) {
1541        RegionBroker region_broker = (RegionBroker) brokerService.getRegionBroker();
1542        Region region;
1543        Collection<Subscription> subs;
1544
1545        region = null;
1546        switch (dest.getDestinationType()) {
1547            case ActiveMQDestination.QUEUE_TYPE:
1548                region = region_broker.getQueueRegion();
1549                break;
1550            case ActiveMQDestination.TOPIC_TYPE:
1551                region = region_broker.getTopicRegion();
1552                break;
1553            case ActiveMQDestination.TEMP_QUEUE_TYPE:
1554                region = region_broker.getTempQueueRegion();
1555                break;
1556            case ActiveMQDestination.TEMP_TOPIC_TYPE:
1557                region = region_broker.getTempTopicRegion();
1558                break;
1559        }
1560
1561        if (region instanceof AbstractRegion) {
1562            subs = ((AbstractRegion) region).getSubscriptions().values();
1563        } else {
1564            subs = null;
1565        }
1566
1567        return subs;
1568    }
1569
1570    protected DemandSubscription createDemandSubscription(ConsumerInfo info) throws IOException {
1571        // add our original id to ourselves
1572        info.addNetworkConsumerId(info.getConsumerId());
1573        return doCreateDemandSubscription(info);
1574    }
1575
1576    protected DemandSubscription doCreateDemandSubscription(ConsumerInfo info) throws IOException {
1577        DemandSubscription result = new DemandSubscription(info);
1578        result.getLocalInfo().setConsumerId(new ConsumerId(localSessionInfo.getSessionId(), consumerIdGenerator.getNextSequenceId()));
1579        if (info.getDestination().isTemporary()) {
1580            // reset the local connection Id
1581            ActiveMQTempDestination dest = (ActiveMQTempDestination) result.getLocalInfo().getDestination();
1582            dest.setConnectionId(localConnectionInfo.getConnectionId().toString());
1583        }
1584
1585        if (configuration.isDecreaseNetworkConsumerPriority()) {
1586            byte priority = (byte) configuration.getConsumerPriorityBase();
1587            if (info.getBrokerPath() != null && info.getBrokerPath().length > 1) {
1588                // The longer the path to the consumer, the less it's consumer priority.
1589                priority -= info.getBrokerPath().length + 1;
1590            }
1591            result.getLocalInfo().setPriority(priority);
1592            LOG.debug("{} using priority: {} for subscription: {}", new Object[]{configuration.getBrokerName(), priority, info});
1593        }
1594        configureDemandSubscription(info, result);
1595        return result;
1596    }
1597
1598    final protected DemandSubscription createDemandSubscription(ActiveMQDestination destination, final String subscriptionName) {
1599        ConsumerInfo info = new ConsumerInfo();
1600        info.setNetworkSubscription(true);
1601        info.setDestination(destination);
1602
1603        if (subscriptionName != null) {
1604            info.setSubscriptionName(subscriptionName);
1605        }
1606
1607        // Indicate that this subscription is being made on behalf of the remote broker.
1608        info.setBrokerPath(new BrokerId[]{remoteBrokerId});
1609
1610        // the remote info held by the DemandSubscription holds the original
1611        // consumerId, the local info get's overwritten
1612        info.setConsumerId(new ConsumerId(localSessionInfo.getSessionId(), consumerIdGenerator.getNextSequenceId()));
1613        DemandSubscription result = null;
1614        try {
1615            result = createDemandSubscription(info);
1616        } catch (IOException e) {
1617            LOG.error("Failed to create DemandSubscription ", e);
1618        }
1619        return result;
1620    }
1621
1622    protected void configureDemandSubscription(ConsumerInfo info, DemandSubscription sub) throws IOException {
1623        if (AdvisorySupport.isConsumerAdvisoryTopic(info.getDestination()) ||
1624                AdvisorySupport.isVirtualDestinationConsumerAdvisoryTopic(info.getDestination())) {
1625            sub.getLocalInfo().setDispatchAsync(true);
1626        } else {
1627            sub.getLocalInfo().setDispatchAsync(configuration.isDispatchAsync());
1628        }
1629        configureConsumerPrefetch(sub.getLocalInfo());
1630        subscriptionMapByLocalId.put(sub.getLocalInfo().getConsumerId(), sub);
1631        subscriptionMapByRemoteId.put(sub.getRemoteInfo().getConsumerId(), sub);
1632
1633        sub.setNetworkBridgeFilter(createNetworkBridgeFilter(info));
1634        if (!info.isDurable()) {
1635            // This works for now since we use a VM connection to the local broker.
1636            // may need to change if we ever subscribe to a remote broker.
1637            sub.getLocalInfo().setAdditionalPredicate(sub.getNetworkBridgeFilter());
1638        } else {
1639            sub.setLocalDurableSubscriber(new SubscriptionInfo(info.getClientId(), info.getSubscriptionName()));
1640        }
1641    }
1642
1643    protected void removeDemandSubscription(ConsumerId id) throws IOException {
1644        DemandSubscription sub = subscriptionMapByRemoteId.remove(id);
1645        LOG.debug("{} remove request on {} from {}, consumer id: {}, matching sub: {}", new Object[]{
1646                configuration.getBrokerName(), localBroker, remoteBrokerName, id, sub
1647        });
1648        if (sub != null) {
1649            removeSubscription(sub);
1650            LOG.debug("{} removed sub on {} from {}: {}", new Object[]{
1651                    configuration.getBrokerName(), localBroker, remoteBrokerName, sub.getRemoteInfo()
1652            });
1653        }
1654    }
1655
1656    protected boolean removeDemandSubscriptionByLocalId(ConsumerId consumerId) {
1657        boolean removeDone = false;
1658        DemandSubscription sub = subscriptionMapByLocalId.get(consumerId);
1659        if (sub != null) {
1660            try {
1661                removeDemandSubscription(sub.getRemoteInfo().getConsumerId());
1662                removeDone = true;
1663            } catch (IOException e) {
1664                LOG.debug("removeDemandSubscriptionByLocalId failed for localId: {}", consumerId, e);
1665            }
1666        }
1667        return removeDone;
1668    }
1669
1670    /**
1671     * Performs a timed wait on the started latch and then checks for disposed
1672     * before performing another wait each time the the started wait times out.
1673     */
1674    protected boolean safeWaitUntilStarted() throws InterruptedException {
1675        while (!disposed.get()) {
1676            if (startedLatch.await(1, TimeUnit.SECONDS)) {
1677                break;
1678            }
1679        }
1680        return !disposed.get();
1681    }
1682
1683    protected NetworkBridgeFilter createNetworkBridgeFilter(ConsumerInfo info) throws IOException {
1684        NetworkBridgeFilterFactory filterFactory = defaultFilterFactory;
1685        if (brokerService != null && brokerService.getDestinationPolicy() != null) {
1686            PolicyEntry entry = brokerService.getDestinationPolicy().getEntryFor(info.getDestination());
1687            if (entry != null && entry.getNetworkBridgeFilterFactory() != null) {
1688                filterFactory = entry.getNetworkBridgeFilterFactory();
1689            }
1690        }
1691        return filterFactory.create(info, getRemoteBrokerPath(), configuration.getMessageTTL(), configuration.getConsumerTTL());
1692    }
1693
1694    protected void addRemoteBrokerToBrokerPath(ConsumerInfo info) throws IOException {
1695        info.setBrokerPath(appendToBrokerPath(info.getBrokerPath(), getRemoteBrokerPath()));
1696    }
1697
1698    protected BrokerId[] getRemoteBrokerPath() {
1699        return remoteBrokerPath;
1700    }
1701
1702    @Override
1703    public void setNetworkBridgeListener(NetworkBridgeListener listener) {
1704        this.networkBridgeListener = listener;
1705    }
1706
1707    private void fireBridgeFailed(Throwable reason) {
1708        LOG.trace("fire bridge failed, listener: {}", this.networkBridgeListener, reason);
1709        NetworkBridgeListener l = this.networkBridgeListener;
1710        if (l != null && this.bridgeFailed.compareAndSet(false, true)) {
1711            l.bridgeFailed();
1712        }
1713    }
1714
1715    /**
1716     * @return Returns the dynamicallyIncludedDestinations.
1717     */
1718    public ActiveMQDestination[] getDynamicallyIncludedDestinations() {
1719        return dynamicallyIncludedDestinations;
1720    }
1721
1722    /**
1723     * @param dynamicallyIncludedDestinations
1724     *         The dynamicallyIncludedDestinations to set.
1725     */
1726    public void setDynamicallyIncludedDestinations(ActiveMQDestination[] dynamicallyIncludedDestinations) {
1727        this.dynamicallyIncludedDestinations = dynamicallyIncludedDestinations;
1728    }
1729
1730    /**
1731     * @return Returns the excludedDestinations.
1732     */
1733    public ActiveMQDestination[] getExcludedDestinations() {
1734        return excludedDestinations;
1735    }
1736
1737    /**
1738     * @param excludedDestinations The excludedDestinations to set.
1739     */
1740    public void setExcludedDestinations(ActiveMQDestination[] excludedDestinations) {
1741        this.excludedDestinations = excludedDestinations;
1742    }
1743
1744    /**
1745     * @return Returns the staticallyIncludedDestinations.
1746     */
1747    public ActiveMQDestination[] getStaticallyIncludedDestinations() {
1748        return staticallyIncludedDestinations;
1749    }
1750
1751    /**
1752     * @param staticallyIncludedDestinations The staticallyIncludedDestinations to set.
1753     */
1754    public void setStaticallyIncludedDestinations(ActiveMQDestination[] staticallyIncludedDestinations) {
1755        this.staticallyIncludedDestinations = staticallyIncludedDestinations;
1756    }
1757
1758    /**
1759     * @return Returns the durableDestinations.
1760     */
1761    public ActiveMQDestination[] getDurableDestinations() {
1762        return durableDestinations;
1763    }
1764
1765    /**
1766     * @param durableDestinations The durableDestinations to set.
1767     */
1768    public void setDurableDestinations(ActiveMQDestination[] durableDestinations) {
1769        this.durableDestinations = durableDestinations;
1770    }
1771
1772    /**
1773     * @return Returns the localBroker.
1774     */
1775    public Transport getLocalBroker() {
1776        return localBroker;
1777    }
1778
1779    /**
1780     * @return Returns the remoteBroker.
1781     */
1782    public Transport getRemoteBroker() {
1783        return remoteBroker;
1784    }
1785
1786    /**
1787     * @return the createdByDuplex
1788     */
1789    public boolean isCreatedByDuplex() {
1790        return this.createdByDuplex;
1791    }
1792
1793    /**
1794     * @param createdByDuplex the createdByDuplex to set
1795     */
1796    public void setCreatedByDuplex(boolean createdByDuplex) {
1797        this.createdByDuplex = createdByDuplex;
1798    }
1799
1800    @Override
1801    public String getRemoteAddress() {
1802        return remoteBroker.getRemoteAddress();
1803    }
1804
1805    @Override
1806    public String getLocalAddress() {
1807        return localBroker.getRemoteAddress();
1808    }
1809
1810    @Override
1811    public String getRemoteBrokerName() {
1812        return remoteBrokerInfo == null ? null : remoteBrokerInfo.getBrokerName();
1813    }
1814
1815    @Override
1816    public String getRemoteBrokerId() {
1817        return (remoteBrokerInfo == null || remoteBrokerInfo.getBrokerId() == null) ? null : remoteBrokerInfo.getBrokerId().toString();
1818    }
1819
1820    @Override
1821    public String getLocalBrokerName() {
1822        return localBrokerInfo == null ? null : localBrokerInfo.getBrokerName();
1823    }
1824
1825    @Override
1826    public long getDequeueCounter() {
1827        return networkBridgeStatistics.getDequeues().getCount();
1828    }
1829
1830    @Override
1831    public long getEnqueueCounter() {
1832        return networkBridgeStatistics.getEnqueues().getCount();
1833    }
1834
1835    @Override
1836    public NetworkBridgeStatistics getNetworkBridgeStatistics() {
1837        return networkBridgeStatistics;
1838    }
1839
1840    protected boolean isDuplex() {
1841        return configuration.isDuplex() || createdByDuplex;
1842    }
1843
1844    public ConcurrentMap<ConsumerId, DemandSubscription> getLocalSubscriptionMap() {
1845        return subscriptionMapByRemoteId;
1846    }
1847
1848    @Override
1849    public void setBrokerService(BrokerService brokerService) {
1850        this.brokerService = brokerService;
1851        this.localBrokerId = brokerService.getRegionBroker().getBrokerId();
1852        localBrokerPath[0] = localBrokerId;
1853    }
1854
1855    @Override
1856    public void setMbeanObjectName(ObjectName objectName) {
1857        this.mbeanObjectName = objectName;
1858    }
1859
1860    @Override
1861    public ObjectName getMbeanObjectName() {
1862        return mbeanObjectName;
1863    }
1864
1865    @Override
1866    public void resetStats() {
1867        networkBridgeStatistics.reset();
1868    }
1869
1870    /*
1871     * Used to allow for async tasks to await receipt of the BrokerInfo from the local and
1872     * remote sides of the network bridge.
1873     */
1874    private static class FutureBrokerInfo implements Future<BrokerInfo> {
1875
1876        private final CountDownLatch slot = new CountDownLatch(1);
1877        private final AtomicBoolean disposed;
1878        private volatile BrokerInfo info = null;
1879
1880        public FutureBrokerInfo(BrokerInfo info, AtomicBoolean disposed) {
1881            this.info = info;
1882            this.disposed = disposed;
1883        }
1884
1885        @Override
1886        public boolean cancel(boolean mayInterruptIfRunning) {
1887            slot.countDown();
1888            return true;
1889        }
1890
1891        @Override
1892        public boolean isCancelled() {
1893            return slot.getCount() == 0 && info == null;
1894        }
1895
1896        @Override
1897        public boolean isDone() {
1898            return info != null;
1899        }
1900
1901        @Override
1902        public BrokerInfo get() throws InterruptedException, ExecutionException {
1903            try {
1904                if (info == null) {
1905                    while (!disposed.get()) {
1906                        if (slot.await(1, TimeUnit.SECONDS)) {
1907                            break;
1908                        }
1909                    }
1910                }
1911                return info;
1912            } catch (InterruptedException e) {
1913                Thread.currentThread().interrupt();
1914                LOG.debug("Operation interrupted: {}", e, e);
1915                throw new InterruptedException("Interrupted.");
1916            }
1917        }
1918
1919        @Override
1920        public BrokerInfo get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
1921            try {
1922                if (info == null) {
1923                    long deadline = System.currentTimeMillis() + unit.toMillis(timeout);
1924
1925                    while (!disposed.get() || System.currentTimeMillis() - deadline < 0) {
1926                        if (slot.await(1, TimeUnit.MILLISECONDS)) {
1927                            break;
1928                        }
1929                    }
1930                    if (info == null) {
1931                        throw new TimeoutException();
1932                    }
1933                }
1934                return info;
1935            } catch (InterruptedException e) {
1936                throw new InterruptedException("Interrupted.");
1937            }
1938        }
1939
1940        public void set(BrokerInfo info) {
1941            this.info = info;
1942            this.slot.countDown();
1943        }
1944    }
1945
1946    protected void serviceOutbound(Message message) {
1947        NetworkBridgeListener l = this.networkBridgeListener;
1948        if (l != null) {
1949            l.onOutboundMessage(this, message);
1950        }
1951    }
1952
1953    protected void serviceInboundMessage(Message message) {
1954        NetworkBridgeListener l = this.networkBridgeListener;
1955        if (l != null) {
1956            l.onInboundMessage(this, message);
1957        }
1958    }
1959
1960    protected boolean canDuplexDispatch(Message message) {
1961        boolean result = true;
1962        if (configuration.isCheckDuplicateMessagesOnDuplex()){
1963            final long producerSequenceId = message.getMessageId().getProducerSequenceId();
1964            //  messages are multiplexed on this producer so we need to query the persistenceAdapter
1965            long lastStoredForMessageProducer = getStoredSequenceIdForMessage(message.getMessageId());
1966            if (producerSequenceId <= lastStoredForMessageProducer) {
1967                result = false;
1968                LOG.debug("suppressing duplicate message send [{}] from network producer with producerSequence [{}] less than last stored: {}", new Object[]{
1969                        (LOG.isTraceEnabled() ? message : message.getMessageId()), producerSequenceId, lastStoredForMessageProducer
1970                });
1971            }
1972        }
1973        return result;
1974    }
1975
1976    protected long getStoredSequenceIdForMessage(MessageId messageId) {
1977        try {
1978            return brokerService.getPersistenceAdapter().getLastProducerSequenceId(messageId.getProducerId());
1979        } catch (IOException ignored) {
1980            LOG.debug("Failed to determine last producer sequence id for: {}", messageId, ignored);
1981        }
1982        return -1;
1983    }
1984
1985    protected void configureConsumerPrefetch(ConsumerInfo consumerInfo) {
1986        //If a consumer on an advisory topic and advisoryPrefetchSize has been explicitly
1987        //set then use it, else default to the prefetchSize setting
1988        if (AdvisorySupport.isAdvisoryTopic(consumerInfo.getDestination()) &&
1989                configuration.getAdvisoryPrefetchSize() > 0) {
1990            consumerInfo.setPrefetchSize(configuration.getAdvisoryPrefetchSize());
1991        } else {
1992            consumerInfo.setPrefetchSize(configuration.getPrefetchSize());
1993        }
1994    }
1995
1996}