org.apache.pulsar.broker.service

Class BrokerService

java.lang.Object

org.apache.pulsar.broker.service.BrokerService

All Implemented Interfaces:

Closeable, AutoCloseable

public class BrokerService
extends Object
implements Closeable

Field Summary

Fields

Modifier and Type	Field and Description
protected PublishRateLimiter	brokerPublishRateLimiter
protected AtomicReference<Semaphore>	lookupRequestSemaphore
static String	MANAGED_LEDGER_PATH_ZNODE
int	maxUnackedMsgsPerDispatcher
static String	PRODUCER_NAME_GENERATOR_PATH
protected AtomicReference<Semaphore>	topicLoadRequestSemaphore

Constructor Summary

Constructors

Constructor and Description
BrokerService(PulsarService pulsar, io.netty.channel.EventLoopGroup eventLoopGroup)

Method Summary

Modifier and Type	Method and Description
void	addUnAckedMessages(PersistentDispatcherMultipleConsumers dispatcher, int numberOfMessages) If per-broker unacked message reached to limit then it blocks dispatcher if its unacked message limit has been reached to maxUnackedMsgsPerDispatcher.
void	cacheOfflineTopicStats(org.apache.pulsar.common.naming.TopicName topicName, org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats offlineTopicStats)
void	checkBrokerPublishThrottlingRate()
void	checkCompaction()
void	checkGC()
void	checkInactiveSubscriptions()
void	checkMessageDeduplicationInfo()
void	checkMessageExpiry()
void	checkReplicationPolicies()
CompletableFuture<Void>	checkTopicNsOwnership(String topic)
void	checkTopicPublishThrottlingRate()
void	checkUnAckMessageDispatching() Adds given dispatcher's unackMessage count to broker-unack message count and if it reaches to the maxUnackedMessages then it blocks all the dispatchers which has unack-messages higher than maxUnackedMsgsPerDispatcher.
void	cleanUnloadedTopicFromCache(NamespaceBundle serviceUnit)
void	close()
CompletableFuture<Void>	closeAndRemoveReplicationClient(String clusterName)
CompletableFuture<Void>	closeAsync()
CompletableFuture<org.apache.pulsar.common.protocol.schema.SchemaVersion>	deleteSchemaStorage(String topic)
CompletableFuture<Void>	deleteTopic(String topic, boolean forceDelete)
CompletableFuture<Void>	deleteTopic(String topic, boolean forceDelete, boolean deleteSchema)
void	deleteTopicAuthenticationWithRetry(String topic, CompletableFuture<Void> future, int count)
CompletableFuture<Void>	deleteTopicPolicies(org.apache.pulsar.common.naming.TopicName topicName)
io.netty.channel.EventLoopGroup	executor()
static Optional<Topic>	extractTopic(CompletableFuture<Optional<Topic>> topicFuture) Safely extract optional topic instance from a future, in a way to avoid unchecked exceptions and race conditions.
CompletableFuture<org.apache.pulsar.common.partition.PartitionedTopicMetadata>	fetchPartitionedTopicMetadataAsync(org.apache.pulsar.common.naming.TopicName topicName)
CompletableFuture<org.apache.pulsar.common.partition.PartitionedTopicMetadata>	fetchPartitionedTopicMetadataCheckAllowAutoCreationAsync(org.apache.pulsar.common.naming.TopicName topicName)
void	forEachTopic(Consumer<Topic> consumer) Iterates over all loaded topics in the broker.
String	generateUniqueProducerName()
List<Topic>	getAllTopicsFromNamespaceBundle(String namespace, String bundle)
AuthenticationService	getAuthenticationService()
AuthorizationService	getAuthorizationService()
BacklogQuotaManager	getBacklogQuotaManager()
Set<org.apache.pulsar.common.intercept.BrokerEntryMetadataInterceptor>	getBrokerEntryMetadataInterceptors()
Map<String,org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats>	getBundleStats()
PulsarAdmin	getClusterPulsarAdmin(String cluster, Optional<org.apache.pulsar.common.policies.data.ClusterData> clusterDataOp)
int	getDefaultNumPartitions(org.apache.pulsar.common.naming.TopicName topicName)
DelayedDeliveryTrackerFactory	getDelayedDeliveryTrackerFactory()
void	getDimensionMetrics(Consumer<io.netty.buffer.ByteBuf> consumer)
static List<String>	getDynamicConfiguration()
int	getKeepAliveIntervalSeconds()
Optional<Integer>	getListenPort()
Optional<Integer>	getListenPortTls()
Semaphore	getLookupRequestSemaphore()
CompletableFuture<ManagedLedgerConfig>	getManagedLedgerConfig(org.apache.pulsar.common.naming.TopicName topicName)
org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap<String,org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap<String,org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap<String,Topic>>>	getMultiLayerTopicMap()
int	getNumberOfNamespaceBundles()
org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats	getOfflineTopicStat(org.apache.pulsar.common.naming.TopicName topicName)
CompletableFuture<Topic>	getOrCreateTopic(String topic)
long	getPausedConnections()
org.apache.pulsar.client.api.PulsarClient	getReplicationClient(String cluster, Optional<org.apache.pulsar.common.policies.data.ClusterData> clusterDataOp)
org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap<String,org.apache.pulsar.client.api.PulsarClient>	getReplicationClients()
Map<String,String>	getRuntimeConfiguration()
CompletableFuture<Optional<Topic>>	getTopic(String topic, boolean createIfMissing)
CompletableFuture<Optional<Topic>>	getTopicIfExists(String topic)
List<org.apache.pulsar.common.stats.Metrics>	getTopicMetrics()
org.apache.bookkeeper.common.util.OrderedExecutor	getTopicOrderedExecutor()
Optional<org.apache.pulsar.common.policies.data.TopicPolicies>	getTopicPolicies(org.apache.pulsar.common.naming.TopicName topicName) Get TopicPolicies for the parameterized topic.
Optional<Topic>	getTopicReference(String topic) Get a reference to a topic that is currently loaded in the broker.
org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap<String,CompletableFuture<Optional<Topic>>>	getTopics()
Map<String,org.apache.pulsar.common.policies.data.stats.TopicStatsImpl>	getTopicStats()
void	invalidateOfflineTopicStatCache(org.apache.pulsar.common.naming.TopicName topicName)
boolean	isAllowAutoSubscriptionCreation(String topic)
boolean	isAllowAutoSubscriptionCreation(org.apache.pulsar.common.naming.TopicName topicName)
boolean	isAllowAutoTopicCreation(String topic)
boolean	isAllowAutoTopicCreation(org.apache.pulsar.common.naming.TopicName topicName)
boolean	isAuthenticationEnabled()
boolean	isAuthorizationEnabled()
boolean	isBrokerDispatchingBlocked()
boolean	isBrokerEntryMetadataEnabled()
boolean	isDefaultTopicTypePartitioned(org.apache.pulsar.common.naming.TopicName topicName)
static boolean	isDynamicConfiguration(String key)
boolean	isSystemTopic(String topic)
boolean	isSystemTopic(org.apache.pulsar.common.naming.TopicName topicName)
boolean	isTopicNsOwnedByBroker(org.apache.pulsar.common.naming.TopicName topicName)
protected CompletableFuture<Optional<Topic>>	loadOrCreatePersistentTopic(String topic, boolean createIfMissing) It creates a topic async and returns CompletableFuture.
void	monitorBacklogQuota()
void	pausedConnections(int numberOfConnections)
PulsarService	pulsar()
void	refreshTopicToStatsMaps(NamespaceBundle oldBundle)
<T> void	registerConfigurationListener(String configKey, Consumer<T> listener) Allows a listener to listen on update of ServiceConfiguration change, so listener can take appropriate action if any specific config-field value has been changed.
CompletableFuture<Void>	removeTopicFromCache(String topic)
void	removeTopicFromCache(String topic, NamespaceBundle namespaceBundle)
void	resumedConnections(int numberOfConnections)
void	setInterceptor(BrokerInterceptor interceptor)
void	setPulsarChannelInitializerFactory(PulsarChannelInitializer.Factory factory)
void	setupBrokerPublishRateLimiterMonitor() Schedules and monitors publish-throttling for broker that has publish-throttling configured.
void	setupTopicPublishRateLimiterMonitor() Schedules and monitors publish-throttling for all owned topics that has publish-throttling configured.
void	start()
protected void	startBacklogQuotaChecker()
protected void	startCheckReplicationPolicies()
protected void	startCompactionMonitor()
protected void	startConsumedLedgersMonitor()
protected void	startDeduplicationSnapshotMonitor()
protected void	startInactivityMonitor()
protected void	startMessageExpiryMonitor()
void	startProtocolHandlers(Map<String,Map<InetSocketAddress,io.netty.channel.ChannelInitializer<io.netty.channel.socket.SocketChannel>>> protocolHandlers)
protected void	startStatsUpdater(int statsUpdateInitialDelayInSecs, int statsUpdateFrequencyInSecs)
void	unblockDispatchersOnUnAckMessages(List<PersistentDispatcherMultipleConsumers> dispatcherList) Unblocks the dispatchers and removes it from the blockedDispatchers list.
void	unloadNamespaceBundlesGracefully() It unloads all owned namespacebundles gracefully.
CompletableFuture<Integer>	unloadServiceUnit(NamespaceBundle serviceUnit, boolean closeWithoutWaitingClientDisconnect, long timeout, TimeUnit unit)
void	updateRates()
static boolean	validateDynamicConfiguration(String key, String value)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

lookupRequestSemaphore

protected final AtomicReference<Semaphore> lookupRequestSemaphore

topicLoadRequestSemaphore

protected final AtomicReference<Semaphore> topicLoadRequestSemaphore

brokerPublishRateLimiter

protected volatile PublishRateLimiter brokerPublishRateLimiter

PRODUCER_NAME_GENERATOR_PATH

public static final String PRODUCER_NAME_GENERATOR_PATH

See Also:

Constant Field Values

MANAGED_LEDGER_PATH_ZNODE

public static final String MANAGED_LEDGER_PATH_ZNODE

See Also:

Constant Field Values

maxUnackedMsgsPerDispatcher

public final int maxUnackedMsgsPerDispatcher

Constructor Detail

BrokerService

public BrokerService(PulsarService pulsar,
                     io.netty.channel.EventLoopGroup eventLoopGroup)
              throws Exception

Throws:

Exception

Method Detail

startProtocolHandlers

public void startProtocolHandlers(Map<String,Map<InetSocketAddress,io.netty.channel.ChannelInitializer<io.netty.channel.socket.SocketChannel>>> protocolHandlers)

start

public void start()
           throws Exception

Throws:

Exception

startStatsUpdater

protected void startStatsUpdater(int statsUpdateInitialDelayInSecs,
                                 int statsUpdateFrequencyInSecs)

startDeduplicationSnapshotMonitor

protected void startDeduplicationSnapshotMonitor()

startInactivityMonitor

protected void startInactivityMonitor()

startMessageExpiryMonitor

protected void startMessageExpiryMonitor()

startCheckReplicationPolicies

protected void startCheckReplicationPolicies()

startCompactionMonitor

protected void startCompactionMonitor()

startConsumedLedgersMonitor

protected void startConsumedLedgersMonitor()

startBacklogQuotaChecker

protected void startBacklogQuotaChecker()

setupTopicPublishRateLimiterMonitor

public void setupTopicPublishRateLimiterMonitor()

Schedules and monitors publish-throttling for all owned topics that has publish-throttling configured. It also disables and shutdowns publish-rate-limiter monitor task if broker disables it.

setupBrokerPublishRateLimiterMonitor

public void setupBrokerPublishRateLimiterMonitor()

Schedules and monitors publish-throttling for broker that has publish-throttling configured. It also disables and shutdowns publish-rate-limiter monitor for broker task if broker disables it.

close

public void close()
           throws IOException

Specified by:

close in interface Closeable

Specified by:

close in interface AutoCloseable

Throws:

IOException

closeAndRemoveReplicationClient

public CompletableFuture<Void> closeAndRemoveReplicationClient(String clusterName)

closeAsync

public CompletableFuture<Void> closeAsync()

unloadNamespaceBundlesGracefully

public void unloadNamespaceBundlesGracefully()

It unloads all owned namespacebundles gracefully.

• First it makes current broker unavailable and isolates from the clusters so, it will not serve any new requests.
• Second it starts unloading namespace bundle one by one without closing the connection in order to avoid disruption for other namespacebundles which are sharing the same connection from the same client.

getTopicIfExists

public CompletableFuture<Optional<Topic>> getTopicIfExists(String topic)

getOrCreateTopic

public CompletableFuture<Topic> getOrCreateTopic(String topic)

getTopic

public CompletableFuture<Optional<Topic>> getTopic(String topic,
                                                   boolean createIfMissing)

deleteSchemaStorage

public CompletableFuture<org.apache.pulsar.common.protocol.schema.SchemaVersion> deleteSchemaStorage(String topic)

deleteTopic

public CompletableFuture<Void> deleteTopic(String topic,
                                           boolean forceDelete)

deleteTopic

public CompletableFuture<Void> deleteTopic(String topic,
                                           boolean forceDelete,
                                           boolean deleteSchema)

deleteTopicAuthenticationWithRetry

public void deleteTopicAuthenticationWithRetry(String topic,
                                               CompletableFuture<Void> future,
                                               int count)

getReplicationClient

public org.apache.pulsar.client.api.PulsarClient getReplicationClient(String cluster,
                                                                      Optional<org.apache.pulsar.common.policies.data.ClusterData> clusterDataOp)

getClusterPulsarAdmin

public PulsarAdmin getClusterPulsarAdmin(String cluster,
                                         Optional<org.apache.pulsar.common.policies.data.ClusterData> clusterDataOp)

loadOrCreatePersistentTopic

protected CompletableFuture<Optional<Topic>> loadOrCreatePersistentTopic(String topic,
                                                                         boolean createIfMissing)
                                                                  throws RuntimeException

It creates a topic async and returns CompletableFuture. It also throttles down configured max-concurrent topic loading and puts them into queue once in-process topics are created.

Parameters:

topic - persistent-topic name

Returns:

CompletableFuture

Throws:

RuntimeException

getManagedLedgerConfig

public CompletableFuture<ManagedLedgerConfig> getManagedLedgerConfig(org.apache.pulsar.common.naming.TopicName topicName)

refreshTopicToStatsMaps

public void refreshTopicToStatsMaps(NamespaceBundle oldBundle)

getOfflineTopicStat

public org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats getOfflineTopicStat(org.apache.pulsar.common.naming.TopicName topicName)

cacheOfflineTopicStats

public void cacheOfflineTopicStats(org.apache.pulsar.common.naming.TopicName topicName,
                                   org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats offlineTopicStats)

invalidateOfflineTopicStatCache

public void invalidateOfflineTopicStatCache(org.apache.pulsar.common.naming.TopicName topicName)

getTopicReference

public Optional<Topic> getTopicReference(String topic)

Get a reference to a topic that is currently loaded in the broker. This method will not make the broker attempt to load the topic if it's not already.

updateRates

public void updateRates()

getDimensionMetrics

public void getDimensionMetrics(Consumer<io.netty.buffer.ByteBuf> consumer)

getTopicMetrics

public List<org.apache.pulsar.common.stats.Metrics> getTopicMetrics()

getBundleStats

public Map<String,org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats> getBundleStats()

getLookupRequestSemaphore

public Semaphore getLookupRequestSemaphore()

checkGC

public void checkGC()

checkMessageExpiry

public void checkMessageExpiry()

checkReplicationPolicies

public void checkReplicationPolicies()

checkCompaction

public void checkCompaction()

checkMessageDeduplicationInfo

public void checkMessageDeduplicationInfo()

checkInactiveSubscriptions

public void checkInactiveSubscriptions()

checkTopicPublishThrottlingRate

public void checkTopicPublishThrottlingRate()

checkBrokerPublishThrottlingRate

public void checkBrokerPublishThrottlingRate()

forEachTopic

public void forEachTopic(Consumer<Topic> consumer)

Iterates over all loaded topics in the broker.

getBacklogQuotaManager

public BacklogQuotaManager getBacklogQuotaManager()

monitorBacklogQuota

public void monitorBacklogQuota()

isTopicNsOwnedByBroker

public boolean isTopicNsOwnedByBroker(org.apache.pulsar.common.naming.TopicName topicName)

checkTopicNsOwnership

public CompletableFuture<Void> checkTopicNsOwnership(String topic)

unloadServiceUnit

public CompletableFuture<Integer> unloadServiceUnit(NamespaceBundle serviceUnit,
                                                    boolean closeWithoutWaitingClientDisconnect,
                                                    long timeout,
                                                    TimeUnit unit)

cleanUnloadedTopicFromCache

public void cleanUnloadedTopicFromCache(NamespaceBundle serviceUnit)

getAuthorizationService

public AuthorizationService getAuthorizationService()

removeTopicFromCache

public CompletableFuture<Void> removeTopicFromCache(String topic)

removeTopicFromCache

public void removeTopicFromCache(String topic,
                                 NamespaceBundle namespaceBundle)

getNumberOfNamespaceBundles

public int getNumberOfNamespaceBundles()

getTopics

public org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap<String,CompletableFuture<Optional<Topic>>> getTopics()

pulsar

public PulsarService pulsar()

executor

public io.netty.channel.EventLoopGroup executor()

getReplicationClients

public org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap<String,org.apache.pulsar.client.api.PulsarClient> getReplicationClients()

isAuthenticationEnabled

public boolean isAuthenticationEnabled()

isAuthorizationEnabled

public boolean isAuthorizationEnabled()

getKeepAliveIntervalSeconds

public int getKeepAliveIntervalSeconds()

generateUniqueProducerName

public String generateUniqueProducerName()

getTopicStats

public Map<String,org.apache.pulsar.common.policies.data.stats.TopicStatsImpl> getTopicStats()

getAuthenticationService

public AuthenticationService getAuthenticationService()

getAllTopicsFromNamespaceBundle

public List<Topic> getAllTopicsFromNamespaceBundle(String namespace,
                                                   String bundle)

registerConfigurationListener

public <T> void registerConfigurationListener(String configKey,
                                              Consumer<T> listener)

Allows a listener to listen on update of ServiceConfiguration change, so listener can take appropriate action if any specific config-field value has been changed. On notification, listener should first check if config value has been changed and after taking appropriate action, listener should update config value with new value if it has been changed (so, next time listener can compare values on configMap change).

Type Parameters:

T -

Parameters:

configKey - : configuration field name

listener - : listener which takes appropriate action on config-value change

getDelayedDeliveryTrackerFactory

public DelayedDeliveryTrackerFactory getDelayedDeliveryTrackerFactory()

getDynamicConfiguration

public static List<String> getDynamicConfiguration()

getRuntimeConfiguration

public Map<String,String> getRuntimeConfiguration()

isDynamicConfiguration

public static boolean isDynamicConfiguration(String key)

validateDynamicConfiguration

public static boolean validateDynamicConfiguration(String key,
                                                   String value)

fetchPartitionedTopicMetadataCheckAllowAutoCreationAsync

public CompletableFuture<org.apache.pulsar.common.partition.PartitionedTopicMetadata> fetchPartitionedTopicMetadataCheckAllowAutoCreationAsync(org.apache.pulsar.common.naming.TopicName topicName)

fetchPartitionedTopicMetadataAsync

public CompletableFuture<org.apache.pulsar.common.partition.PartitionedTopicMetadata> fetchPartitionedTopicMetadataAsync(org.apache.pulsar.common.naming.TopicName topicName)

getTopicOrderedExecutor

public org.apache.bookkeeper.common.util.OrderedExecutor getTopicOrderedExecutor()

getMultiLayerTopicMap

public org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap<String,org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap<String,org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap<String,Topic>>> getMultiLayerTopicMap()

addUnAckedMessages

public void addUnAckedMessages(PersistentDispatcherMultipleConsumers dispatcher,
                               int numberOfMessages)

If per-broker unacked message reached to limit then it blocks dispatcher if its unacked message limit has been reached to maxUnackedMsgsPerDispatcher.

Parameters:

dispatcher -

numberOfMessages -

checkUnAckMessageDispatching

public void checkUnAckMessageDispatching()

Adds given dispatcher's unackMessage count to broker-unack message count and if it reaches to the maxUnackedMessages then it blocks all the dispatchers which has unack-messages higher than maxUnackedMsgsPerDispatcher. It unblocks all dispatchers once broker-unack message counts decreased to (maxUnackedMessages/2)

isBrokerDispatchingBlocked

public boolean isBrokerDispatchingBlocked()

unblockDispatchersOnUnAckMessages

public void unblockDispatchersOnUnAckMessages(List<PersistentDispatcherMultipleConsumers> dispatcherList)

Unblocks the dispatchers and removes it from the blockedDispatchers list.

Parameters:

dispatcherList -

extractTopic

public static Optional<Topic> extractTopic(CompletableFuture<Optional<Topic>> topicFuture)

Safely extract optional topic instance from a future, in a way to avoid unchecked exceptions and race conditions.

getListenPort

public Optional<Integer> getListenPort()

getListenPortTls

public Optional<Integer> getListenPortTls()

isAllowAutoTopicCreation

public boolean isAllowAutoTopicCreation(String topic)

isAllowAutoTopicCreation

public boolean isAllowAutoTopicCreation(org.apache.pulsar.common.naming.TopicName topicName)

isDefaultTopicTypePartitioned

public boolean isDefaultTopicTypePartitioned(org.apache.pulsar.common.naming.TopicName topicName)

getDefaultNumPartitions

public int getDefaultNumPartitions(org.apache.pulsar.common.naming.TopicName topicName)

isAllowAutoSubscriptionCreation

public boolean isAllowAutoSubscriptionCreation(String topic)

isAllowAutoSubscriptionCreation

public boolean isAllowAutoSubscriptionCreation(org.apache.pulsar.common.naming.TopicName topicName)

isSystemTopic

public boolean isSystemTopic(String topic)

isSystemTopic

public boolean isSystemTopic(org.apache.pulsar.common.naming.TopicName topicName)

getTopicPolicies

public Optional<org.apache.pulsar.common.policies.data.TopicPolicies> getTopicPolicies(org.apache.pulsar.common.naming.TopicName topicName)

Get TopicPolicies for the parameterized topic.

Parameters:

topicName -

Returns:

TopicPolicies, if they exist. Otherwise, the value will not be present.

deleteTopicPolicies

public CompletableFuture<Void> deleteTopicPolicies(org.apache.pulsar.common.naming.TopicName topicName)

setInterceptor

public void setInterceptor(BrokerInterceptor interceptor)

getBrokerEntryMetadataInterceptors

public Set<org.apache.pulsar.common.intercept.BrokerEntryMetadataInterceptor> getBrokerEntryMetadataInterceptors()

isBrokerEntryMetadataEnabled

public boolean isBrokerEntryMetadataEnabled()

pausedConnections

public void pausedConnections(int numberOfConnections)

resumedConnections

public void resumedConnections(int numberOfConnections)

getPausedConnections

public long getPausedConnections()

setPulsarChannelInitializerFactory

public void setPulsarChannelInitializerFactory(PulsarChannelInitializer.Factory factory)