From b0b74f4e6f38b4c4b02f7173d33f90d0868aea3a Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Wed, 26 Mar 2025 18:01:20 +0800 Subject: [PATCH 01/16] [fix][broker] Add topic consistency check (#24118) Signed-off-by: Zixuan Liu (cherry picked from commit 7f0429c1cd81fbde6e0213a84e42f1ea55b5f7d2) # Conflicts: # pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java # pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java # pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java --- .../broker/namespace/NamespaceService.java | 35 ++-- .../pulsar/broker/service/BrokerService.java | 152 +++++++++--------- .../pulsar/broker/admin/AdminApiTest.java | 6 +- .../broker/admin/TopicAutoCreationTest.java | 20 +-- .../protocol/PulsarClientBasedHandler.java | 7 +- .../broker/service/ExclusiveProducerTest.java | 13 +- .../broker/service/PersistentTopicTest.java | 8 +- .../pulsar/broker/service/ReplicatorTest.java | 6 +- .../nonpersistent/NonPersistentTopicTest.java | 13 +- .../persistent/PersistentTopicTest.java | 31 ---- .../client/api/ConsumerCreationTest.java | 127 +++++++++++++++ .../client/api/ProducerCreationTest.java | 73 +++++++++ .../pulsar/client/impl/LookupServiceTest.java | 82 ++++++++++ .../client/api/PulsarClientException.java | 4 +- 14 files changed, 425 insertions(+), 152 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerCreationTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index b5dbddc2ea377..04b6794d94f29 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -1397,17 +1397,26 @@ public CompletableFuture checkTopicExistsAsync(TopicName topic) */ @Deprecated public CompletableFuture checkTopicExists(TopicName topic) { - return pulsar.getBrokerService() - .fetchPartitionedTopicMetadataAsync(TopicName.get(topic.toString())) - .thenCompose(metadata -> { - if (metadata.partitions > 0) { - return CompletableFuture.completedFuture( - TopicExistsInfo.newPartitionedTopicExists(metadata.partitions)); - } - return checkNonPartitionedTopicExists(topic) - .thenApply(b -> b ? TopicExistsInfo.newNonPartitionedTopicExists() - : TopicExistsInfo.newTopicNotExists()); - }); + // For non-persistent/persistent partitioned topic, which has metadata. + return pulsar.getBrokerService().fetchPartitionedTopicMetadataAsync( + topic.isPartitioned() ? TopicName.get(topic.getPartitionedTopicName()) : topic) + .thenCompose(metadata -> { + if (metadata.partitions > 0) { + if (!topic.isPartitioned()) { + return CompletableFuture.completedFuture( + TopicExistsInfo.newPartitionedTopicExists(metadata.partitions)); + } else { + if (topic.getPartitionIndex() < metadata.partitions) { + return CompletableFuture.completedFuture( + TopicExistsInfo.newNonPartitionedTopicExists()); + } + } + } + // Direct query the single topic. + return checkNonPartitionedTopicExists(topic).thenApply( + b -> b ? TopicExistsInfo.newNonPartitionedTopicExists() : + TopicExistsInfo.newTopicNotExists()); + }); } /*** @@ -1428,12 +1437,12 @@ public CompletableFuture checkNonPartitionedTopicExists(TopicName topic */ public CompletableFuture checkNonPersistentNonPartitionedTopicExists(String topic) { TopicName topicName = TopicName.get(topic); - // "non-partitioned & non-persistent" topics only exist on the owner broker. + // "non-partitioned & non-persistent" topics only exist on the cache of the owner broker. return checkTopicOwnership(TopicName.get(topic)).thenCompose(isOwned -> { // The current broker is the owner. if (isOwned) { CompletableFuture> nonPersistentTopicFuture = pulsar.getBrokerService() - .getTopic(topic, false); + .getTopics().get(topic); if (nonPersistentTopicFuture != null) { return nonPersistentTopicFuture.thenApply(Optional::isPresent); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 7e9819f7f0439..c8018b8c3f12b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1081,6 +1081,70 @@ public CompletableFuture> getTopic(final String topic, boolean c return getTopic(TopicName.get(topic), createIfMissing, properties); } + /** + * Validates that the topic is consistent with its partition metadata. + * + * This method ensures the topic (partitioned or non-partitioned) correctly + * matches the actual partitions in the metadata. Inconsistencies typically + * indicate configuration issues or metadata synchronization problems. + * + * This validation is particularly important in geo-replicated environments where + * topic metadata may not be fully synchronized across all regions, potentially + * leading to access errors if not properly handled. + * + * @param topicName The topic name to validate + * @return CompletableFuture that completes normally if validation passes, or + * completes exceptionally with NotAllowedException if validation fails + */ + private CompletableFuture validateTopicConsistency(TopicName topicName) { + if (NamespaceService.isHeartbeatNamespace(topicName.getNamespaceObject())) { + // Skip validation for heartbeat namespace. + return CompletableFuture.completedFuture(null); + } + TopicName baseTopicName = + topicName.isPartitioned() ? TopicName.get(topicName.getPartitionedTopicName()) : topicName; + return fetchPartitionedTopicMetadataAsync(baseTopicName) + .thenCompose(metadata -> { + if (topicName.isPartitioned()) { + if (metadata.partitions == 0) { + // Edge case: When a complete partitioned topic name is provided but metadata shows 0 + // partitions. + // This indicates that the partitioned topic metadata doesn't exist. + // + // Resolution options: + // 1. Creates the partitioned topic via admin API. + // 2. Uses the base topic name and then rely on auto-creation the partitioned topic if + // enabled. + return FutureUtil.failedFuture( + new BrokerServiceException.NotAllowedException( + "Partition metadata not found for the partitioned topic: " + topicName)); + } + if (topicName.getPartitionIndex() >= metadata.partitions) { + final String errorMsg = + String.format( + "Illegal topic partition name %s with max allowed " + + "%d partitions", topicName, + metadata.partitions); + log.warn(errorMsg); + return FutureUtil.failedFuture( + new BrokerServiceException.NotAllowedException(errorMsg)); + } + } else if (metadata.partitions > 0) { + // Edge case: Non-partitioned topic name was provided, but metadata indicates this is + // actually a partitioned + // topic (partitions > 0). + // + // Resolution: Must use the complete partitioned topic name('topic-name-partition-N'). + // + // This ensures proper routing to the specific partition and prevents ambiguity in topic + // addressing. + return FutureUtil.failedFuture(new BrokerServiceException.NotAllowedException( + "Found partitioned metadata for non-partitioned topic: " + topicName)); + } + return CompletableFuture.completedFuture(null); + }); + } + /** * Retrieves or creates a topic based on the specified parameters. * 0. If disable PersistentTopics or NonPersistentTopics, it will return a failed future with NotAllowedException. @@ -1168,10 +1232,6 @@ public CompletableFuture> getTopic(final TopicName topicName, bo topicFuture.completeExceptionally(rc); return null; }); - }).exceptionally(e -> { - pulsar.getExecutor().execute(() -> topics.remove(topicName.toString(), topicFuture)); - topicFuture.completeExceptionally(e.getCause()); - return null; }); return topicFuture; } else { @@ -1185,29 +1245,10 @@ public CompletableFuture> getTopic(final TopicName topicName, bo if (!topics.containsKey(topicName.toString())) { topicEventsDispatcher.notify(topicName.toString(), TopicEvent.LOAD, EventStage.BEFORE); } - if (topicName.isPartitioned()) { - final TopicName partitionedTopicName = TopicName.get(topicName.getPartitionedTopicName()); - return this.fetchPartitionedTopicMetadataAsync(partitionedTopicName).thenCompose((metadata) -> { - if (topicName.getPartitionIndex() < metadata.partitions) { - return topics.computeIfAbsent(topicName.toString(), (name) -> { - topicEventsDispatcher - .notify(topicName.toString(), TopicEvent.CREATE, EventStage.BEFORE); - - CompletableFuture> res = createNonPersistentTopic(name); - - CompletableFuture> eventFuture = topicEventsDispatcher - .notifyOnCompletion(res, topicName.toString(), TopicEvent.CREATE); - topicEventsDispatcher - .notifyOnCompletion(eventFuture, topicName.toString(), TopicEvent.LOAD); - return res; - }); - } - topicEventsDispatcher.notify(topicName.toString(), TopicEvent.LOAD, EventStage.FAILURE); - return CompletableFuture.completedFuture(Optional.empty()); - }); - } else if (createIfMissing) { + if (topicName.isPartitioned() || createIfMissing) { return topics.computeIfAbsent(topicName.toString(), (name) -> { - topicEventsDispatcher.notify(topicName.toString(), TopicEvent.CREATE, EventStage.BEFORE); + topicEventsDispatcher + .notify(topicName.toString(), TopicEvent.CREATE, EventStage.BEFORE); CompletableFuture> res = createNonPersistentTopic(name); @@ -1217,14 +1258,13 @@ public CompletableFuture> getTopic(final TopicName topicName, bo .notifyOnCompletion(eventFuture, topicName.toString(), TopicEvent.LOAD); return res; }); - } else { - CompletableFuture> topicFuture = topics.get(topicName.toString()); - if (topicFuture == null) { - topicEventsDispatcher.notify(topicName.toString(), TopicEvent.LOAD, EventStage.FAILURE); - topicFuture = CompletableFuture.completedFuture(Optional.empty()); - } - return topicFuture; } + CompletableFuture> topicFuture = topics.get(topicName.toString()); + if (topicFuture == null) { + topicEventsDispatcher.notify(topicName.toString(), TopicEvent.LOAD, EventStage.FAILURE); + topicFuture = CompletableFuture.completedFuture(Optional.empty()); + } + return topicFuture; } } catch (IllegalArgumentException e) { log.warn("[{}] Illegalargument exception when loading topic", topicName, e); @@ -1404,8 +1444,9 @@ private CompletableFuture> createNonPersistentTopic(String topic topicFuture.completeExceptionally(e); return topicFuture; } - CompletableFuture isOwner = checkTopicNsOwnership(topic); - isOwner.thenRun(() -> { + checkTopicNsOwnership(topic) + .thenCompose((__) -> validateTopicConsistency(TopicName.get(topic))) + .thenRun(() -> { nonPersistentTopic.initialize() .thenCompose(__ -> nonPersistentTopic.checkReplication()) .thenRun(() -> { @@ -1422,17 +1463,7 @@ private CompletableFuture> createNonPersistentTopic(String topic return null; }); }).exceptionally(e -> { - log.warn("CheckTopicNsOwnership fail when createNonPersistentTopic! {}", topic, e.getCause()); - // CheckTopicNsOwnership fail dont create nonPersistentTopic, when topic do lookup will find the correct - // broker. When client get non-persistent-partitioned topic - // metadata will the non-persistent-topic will be created. - // so we should add checkTopicNsOwnership logic otherwise the topic will be created - // if it dont own by this broker,we should return success - // otherwise it will keep retrying getPartitionedTopicMetadata - topicFuture.complete(Optional.of(nonPersistentTopic)); - // after get metadata return success, we should delete this topic from this broker, because this topic not - // owner by this broker and it don't initialize and checkReplication - pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); + topicFuture.completeExceptionally(FutureUtil.unwrapCompletionException(e)); return null; }); @@ -1675,32 +1706,9 @@ public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional c * loading and puts them into queue once in-process topics are created. */ protected CompletableFuture> loadOrCreatePersistentTopic(TopicLoadingContext context) { - final var topicName = context.getTopicName(); final var topic = context.getTopicName().toString(); - final CompletableFuture ownedFuture; - if (topicName.isPartitioned()) { - final TopicName topicNameEntity = TopicName.get(topicName.getPartitionedTopicName()); - ownedFuture = fetchPartitionedTopicMetadataAsync(topicNameEntity) - .thenCompose((metadata) -> { - // Allow creating non-partitioned persistent topic that name includes - // `partition` - if (metadata.partitions == 0 - || topicName.getPartitionIndex() < metadata.partitions) { - return checkTopicNsOwnership(topic); - } else { - final String errorMsg = - String.format("Illegal topic partition name %s with max allowed " - + "%d partitions", topicName, metadata.partitions); - log.warn(errorMsg); - return FutureUtil.failedFuture( - new BrokerServiceException.NotAllowedException(errorMsg)); - } - }); - } else { - ownedFuture = checkTopicNsOwnership(topic); - } final var topicFuture = context.getTopicFuture(); - ownedFuture + checkTopicNsOwnership(topic) .thenRun(() -> { final Semaphore topicLoadSemaphore = topicLoadRequestSemaphore.get(); @@ -1812,8 +1820,8 @@ public void createPersistentTopic0(TopicLoadingContext context) { : CompletableFuture.completedFuture(null); CompletableFuture isTopicAlreadyMigrated = checkTopicAlreadyMigrated(topicName); - - maxTopicsCheck.thenCompose(__ -> isTopicAlreadyMigrated) + maxTopicsCheck.thenCompose(partitionedTopicMetadata -> validateTopicConsistency(topicName)) + .thenCompose(__ -> isTopicAlreadyMigrated) .thenCompose(__ -> getManagedLedgerConfig(topicName)) .thenCombine(pulsar().getNamespaceService().checkTopicExistsAsync(topicName).thenApply(n -> { boolean found = n.isExists(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java index 2c5d045f7581b..aa937fb31fb67 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java @@ -3147,10 +3147,8 @@ public void testPersistentTopicsExpireMessages() throws Exception { @Test public void testPersistentTopicsExpireMessagesInvalidPartitionIndex() throws Exception { - // Force to create a topic - publishMessagesOnPersistentTopic("persistent://prop-xyz/ns1/ds2-partition-2", 0); - assertEquals(admin.topics().getList("prop-xyz/ns1"), - List.of("persistent://prop-xyz/ns1/ds2-partition-2")); + // Create a topic + admin.topics().createPartitionedTopic("persistent://prop-xyz/ns1/ds2", 3); // create consumer and subscription @Cleanup diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAutoCreationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAutoCreationTest.java index aa08b2094d8b6..f4b247c3e7ade 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAutoCreationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAutoCreationTest.java @@ -23,6 +23,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; import java.io.Closeable; import java.net.InetSocketAddress; @@ -43,6 +44,7 @@ import org.apache.pulsar.client.api.ProducerConsumerBase; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.PulsarClientException.NotAllowedException; import org.apache.pulsar.client.impl.LookupService; import org.apache.pulsar.client.impl.LookupTopicResult; import org.apache.pulsar.client.impl.PulsarClientImpl; @@ -108,16 +110,14 @@ public void testPartitionedTopicAutoCreation() throws PulsarAdminException, Puls final String partition = "persistent://" + namespaceName + "/test-partitioned-topi-auto-creation-partition-0"; - producer = pulsarClient.newProducer() - .topic(partition) - .create(); - - partitionedTopics = admin.topics().getPartitionedTopicList(namespaceName); - topics = admin.topics().getList(namespaceName); - assertEquals(partitionedTopics.size(), 0); - assertEquals(topics.size(), 1); - - producer.close(); + // The Pulsar doesn't automatically create the metadata for the single partition, so the producer creation + // will fail. + assertThrows(NotAllowedException.class, () -> { + @Cleanup + Producer ignored = pulsarClient.newProducer() + .topic(partition) + .create(); + }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/PulsarClientBasedHandler.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/PulsarClientBasedHandler.java index ed9881a8cadb9..3d24fe3ce38dc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/PulsarClientBasedHandler.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/PulsarClientBasedHandler.java @@ -79,9 +79,11 @@ public String getProtocolDataToAdvertise() { @Override public void start(BrokerService service) { + @Cleanup + PulsarAdmin admin = null; try { final var port = service.getPulsar().getListenPortHTTP().orElseThrow(); - @Cleanup final var admin = PulsarAdmin.builder().serviceHttpUrl("http://localhost:" + port).build(); + admin = PulsarAdmin.builder().serviceHttpUrl("http://localhost:" + port).build(); try { admin.clusters().createCluster(cluster, ClusterData.builder() .serviceUrl(service.getPulsar().getWebServiceAddress()) @@ -103,6 +105,7 @@ public void start(BrokerService service) { throw new RuntimeException(e); } try { + admin.topics().createPartitionedTopic(topic, partitions); final var port = service.getListenPort().orElseThrow(); client = PulsarClient.builder().serviceUrl("pulsar://localhost:" + port).build(); readers = new ArrayList<>(); @@ -122,7 +125,7 @@ public void start(BrokerService service) { }); } }); - } catch (PulsarClientException e) { + } catch (PulsarClientException | PulsarAdminException e) { throw new RuntimeException(e); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ExclusiveProducerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ExclusiveProducerTest.java index fcbcae94bda7e..9a070efa95d4b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ExclusiveProducerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ExclusiveProducerTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.service; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import io.netty.util.HashedWheelTimer; @@ -32,6 +33,7 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException.ProducerBusyException; import org.apache.pulsar.client.api.PulsarClientException.ProducerFencedException; +import org.apache.pulsar.client.api.PulsarClientException.TimeoutException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.naming.TopicName; @@ -344,6 +346,7 @@ public void producerFenced(ProducerAccessMode accessMode, boolean partitioned) t public void topicDeleted(String ignored, boolean partitioned) throws Exception { String topic = newTopic("persistent", partitioned); + @Cleanup Producer p1 = pulsarClient.newProducer(Schema.STRING) .topic(topic) .accessMode(ProducerAccessMode.Exclusive) @@ -357,8 +360,14 @@ public void topicDeleted(String ignored, boolean partitioned) throws Exception { admin.topics().delete(topic, true); } - // The producer should be able to publish again on the topic - p1.send("msg-2"); + if (!partitioned) { + // The producer should be able to publish again on the topic + p1.send("msg-2"); + } else { + // The partitioned topic is deleted, the producer should not be able to publish again on the topic. + // Partitioned metadata is required to publish messages to the topic. + assertThrows(TimeoutException.class, () -> p1.send("msg-2")); + } } @Test(dataProvider = "topics") diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index b6ac7923d2a08..5f124c34658b8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -130,7 +130,6 @@ import org.apache.pulsar.common.api.proto.ProducerAccessMode; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl; @@ -146,7 +145,7 @@ import org.apache.pulsar.compaction.CompactorMXBean; import org.apache.pulsar.compaction.PulsarCompactionServiceFactory; import org.apache.pulsar.metadata.api.MetadataStoreException; -import org.apache.pulsar.metadata.impl.FaultInjectionMetadataStore; +import org.apache.pulsar.metadata.impl.FaultInjectionMetadataStore.OperationType; import org.awaitility.Awaitility; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; @@ -1475,8 +1474,6 @@ public void testDeleteTopicDeleteOnMetadataStoreFailed() throws Exception { doReturn(CompletableFuture.completedFuture(null)).when(ledgerMock).asyncTruncate(); // create topic - brokerService.pulsar().getPulsarResources().getNamespaceResources().getPartitionedTopicResources() - .createPartitionedTopic(TopicName.get(successTopicName), new PartitionedTopicMetadata(2)); PersistentTopic topic = (PersistentTopic) brokerService.getOrCreateTopic(successTopicName).get(); Field isFencedField = AbstractTopic.class.getDeclaredField("isFenced"); @@ -1488,8 +1485,7 @@ public void testDeleteTopicDeleteOnMetadataStoreFailed() throws Exception { assertFalse((boolean) isClosingOrDeletingField.get(topic)); metadataStore.failConditional(new MetadataStoreException("injected error"), (op, path) -> - op == FaultInjectionMetadataStore.OperationType.PUT - && path.equals("/admin/partitioned-topics/prop/use/ns-abc/persistent/successTopic")); + op == OperationType.EXISTS && path.equals("/admin/flags/policies-readonly")); try { topic.delete().get(); fail(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java index 351c965c863db..3802ec459e032 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java @@ -70,7 +70,7 @@ import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; -import org.apache.pulsar.broker.service.BrokerServiceException.NamingException; +import org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException; import org.apache.pulsar.broker.service.persistent.PersistentReplicator; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.stats.OpenTelemetryReplicatorStats; @@ -1227,7 +1227,7 @@ public void testReplicatorOnPartitionedTopic(boolean isPartitionedTopic) throws if (!isPartitionedTopic) { fail("Topic creation should not fail without any partitioned topic"); } - assertTrue(e.getCause() instanceof NamingException); + assertTrue(e.getCause() instanceof NotAllowedException); } // non-persistent topic test @@ -1240,7 +1240,7 @@ public void testReplicatorOnPartitionedTopic(boolean isPartitionedTopic) throws if (!isPartitionedTopic) { fail("Topic creation should not fail without any partitioned topic"); } - assertTrue(e.getCause() instanceof NamingException); + assertTrue(e.getCause() instanceof NotAllowedException); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java index 0acd574bb09b1..d902434f9bdb6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java @@ -20,8 +20,8 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; import java.lang.reflect.Field; import java.util.Optional; import java.util.UUID; @@ -112,19 +112,16 @@ public void testAccumulativeStats() throws Exception { } @Test - public void testCreateNonExistentPartitions() throws PulsarAdminException, PulsarClientException { + public void testCreateNonExistentPartitions() throws PulsarAdminException { final String topicName = "non-persistent://prop/ns-abc/testCreateNonExistentPartitions"; admin.topics().createPartitionedTopic(topicName, 4); TopicName partition = TopicName.get(topicName).getPartition(4); - try { + assertThrows(PulsarClientException.NotAllowedException.class, () -> { @Cleanup - Producer producer = pulsarClient.newProducer() + Producer ignored = pulsarClient.newProducer() .topic(partition.toString()) .create(); - fail("unexpected behaviour"); - } catch (PulsarClientException.NotFoundException ignored) { - - } + }); assertEquals(admin.topics().getPartitionedTopicMetadata(topicName).partitions, 4); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java index 462a5002bd3f2..364360573da70 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java @@ -580,37 +580,6 @@ public void testCreateNonExistentPartitions() throws PulsarAdminException, Pulsa Assert.assertEquals(admin.topics().getPartitionedTopicMetadata(topicName).partitions, 4); } - @Test - public void testCompatibilityWithPartitionKeyword() throws PulsarAdminException, PulsarClientException { - final String topicName = "persistent://prop/ns-abc/testCompatibilityWithPartitionKeyword"; - TopicName topicNameEntity = TopicName.get(topicName); - String partition2 = topicNameEntity.getPartition(2).toString(); - // Create a non-partitioned topic with -partition- keyword - Producer producer = pulsarClient.newProducer() - .topic(partition2) - .create(); - List topics = admin.topics().getList("prop/ns-abc"); - // Close previous producer to simulate reconnect - producer.close(); - // Disable auto topic creation - conf.setAllowAutoTopicCreation(false); - // Check the topic exist in the list. - Assert.assertTrue(topics.contains(partition2)); - // Check this topic has no partition metadata. - Assert.assertThrows(PulsarAdminException.NotFoundException.class, - () -> admin.topics().getPartitionedTopicMetadata(topicName)); - // Reconnect to the broker and expect successful because the topic has existed in the broker. - producer = pulsarClient.newProducer() - .topic(partition2) - .create(); - producer.close(); - // Check the topic exist in the list again. - Assert.assertTrue(topics.contains(partition2)); - // Check this topic has no partition metadata again. - Assert.assertThrows(PulsarAdminException.NotFoundException.class, - () -> admin.topics().getPartitionedTopicMetadata(topicName)); - } - @Test public void testDeleteTopicFail() throws Exception { final String fullyTopicName = "persistent://prop/ns-abc/" + "tp_" diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerCreationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerCreationTest.java new file mode 100644 index 0000000000000..a81dbe02b3447 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerCreationTest.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.client.api; + +import static org.testng.Assert.assertThrows; +import lombok.Cleanup; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.PulsarClientException.NotAllowedException; +import org.apache.pulsar.common.naming.TopicDomain; +import org.apache.pulsar.common.naming.TopicName; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Test(groups = "broker-api") +public class ConsumerCreationTest extends ProducerConsumerBase { + + @BeforeMethod + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @DataProvider(name = "topicDomainProvider") + public Object[][] topicDomainProvider() { + return new Object[][]{ + {TopicDomain.persistent}, + {TopicDomain.non_persistent} + }; + } + + @Test(dataProvider = "topicDomainProvider") + public void testCreateConsumerWhenTopicTypeMismatch(TopicDomain domain) + throws PulsarAdminException, PulsarClientException { + String nonPartitionedTopic = + TopicName.get(domain.value(), "public", "default", + "testCreateConsumerWhenTopicTypeMismatch-nonPartitionedTopic") + .toString(); + admin.topics().createNonPartitionedTopic(nonPartitionedTopic); + + // Topic type is non-partitioned, Trying to create consumer on partitioned topic. + assertThrows(NotAllowedException.class, () -> { + @Cleanup + Consumer ignored = + pulsarClient.newConsumer().topic(TopicName.get(nonPartitionedTopic).getPartition(2).toString()) + .subscriptionName("my-sub").subscribe(); + }); + + // Topic type is partitioned, Trying to create consumer on non-partitioned topic. + String partitionedTopic = TopicName.get(domain.value(), "public", "default", + "testCreateConsumerWhenTopicTypeMismatch-partitionedTopic") + .toString(); + admin.topics().createPartitionedTopic(partitionedTopic, 3); + + // Works fine because the lookup can help our to find the correct topic. + { + @Cleanup + Consumer ignored = + pulsarClient.newConsumer().topic(TopicName.get(partitionedTopic).getPartition(2).toString()) + .subscriptionName("my-sub").subscribe(); + } + + // Partition index is out of range. + assertThrows(NotAllowedException.class, () -> { + @Cleanup + Consumer ignored = + pulsarClient.newConsumer().topic(TopicName.get(partitionedTopic).getPartition(100).toString()) + .subscriptionName("my-sub").subscribe(); + }); + } + + @Test(dataProvider = "topicDomainProvider") + public void testCreateConsumerWhenSinglePartitionIsDeleted(TopicDomain domain) + throws PulsarAdminException, PulsarClientException { + testCreateConsumerWhenSinglePartitionIsDeleted(domain, false); + testCreateConsumerWhenSinglePartitionIsDeleted(domain, true); + } + + private void testCreateConsumerWhenSinglePartitionIsDeleted(TopicDomain domain, boolean allowAutoTopicCreation) + throws PulsarAdminException, PulsarClientException { + conf.setAllowAutoTopicCreation(allowAutoTopicCreation); + + String partitionedTopic = TopicName.get(domain.value(), "public", "default", + "testCreateConsumerWhenSinglePartitionIsDeleted-" + allowAutoTopicCreation) + .toString(); + admin.topics().createPartitionedTopic(partitionedTopic, 3); + admin.topics().delete(TopicName.get(partitionedTopic).getPartition(1).toString()); + + // Non-persistent topic only have the metadata, and no partition, so it works fine. + if (allowAutoTopicCreation || domain.equals(TopicDomain.non_persistent)) { + @Cleanup + Consumer ignored = + pulsarClient.newConsumer().topic(partitionedTopic).subscriptionName("my-sub").subscribe(); + } else { + assertThrows(PulsarClientException.TopicDoesNotExistException.class, () -> { + @Cleanup + Consumer ignored = + pulsarClient.newConsumer().topic(partitionedTopic).subscriptionName("my-sub").subscribe(); + }); + } + } +} \ No newline at end of file diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerCreationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerCreationTest.java index a2dd8a1b8dd74..dee53a5df01f1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerCreationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerCreationTest.java @@ -18,8 +18,11 @@ */ package org.apache.pulsar.client.api; +import static org.testng.Assert.assertThrows; import static org.testng.Assert.fail; +import lombok.Cleanup; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.PulsarClientException.NotAllowedException; import org.apache.pulsar.client.impl.ProducerBuilderImpl; import org.apache.pulsar.client.impl.ProducerImpl; import org.apache.pulsar.common.naming.TopicDomain; @@ -194,4 +197,74 @@ public void testInitialSubscriptionCreationWithAutoCreationDisable() Assert.assertFalse(admin.topics().getSubscriptions(topic.toString()).contains(initialSubscriptionName)); } + + @Test(dataProvider = "topicDomainProvider") + public void testCreateProducerWhenTopicTypeMismatch(TopicDomain domain) + throws PulsarAdminException, PulsarClientException { + String nonPartitionedTopic = + TopicName.get(domain.value(), "public", "default", + "testCreateProducerWhenTopicTypeMismatch-nonPartitionedTopic") + .toString(); + admin.topics().createNonPartitionedTopic(nonPartitionedTopic); + + // Topic type is non-partitioned, trying to create producer on the complete partitioned topic. + // Should throw NotAllowedException. + assertThrows(NotAllowedException.class, () -> { + @Cleanup + Producer ignored = + pulsarClient.newProducer().topic(TopicName.get(nonPartitionedTopic).getPartition(2).toString()) + .create(); + }); + + // Topic type is partitioned, trying to create producer on the base partitioned topic. + String partitionedTopic = TopicName.get(domain.value(), "public", "default", + "testCreateProducerWhenTopicTypeMismatch-partitionedTopic") + .toString(); + admin.topics().createPartitionedTopic(partitionedTopic, 3); + + // Works fine because the lookup can help our to find all the topics. + { + @Cleanup + Producer ignored = + pulsarClient.newProducer().topic(TopicName.get(partitionedTopic).getPartitionedTopicName()) + .create(); + } + + // Partition index is out of range. + assertThrows(NotAllowedException.class, () -> { + @Cleanup + Producer ignored = + pulsarClient.newProducer().topic(TopicName.get(partitionedTopic).getPartition(100).toString()) + .create(); + }); + } + + @Test(dataProvider = "topicDomainProvider") + public void testCreateProducerWhenSinglePartitionIsDeleted(TopicDomain domain) + throws PulsarAdminException, PulsarClientException { + testCreateProducerWhenSinglePartitionIsDeleted(domain, false); + testCreateProducerWhenSinglePartitionIsDeleted(domain, true); + } + + private void testCreateProducerWhenSinglePartitionIsDeleted(TopicDomain domain, boolean allowAutoTopicCreation) + throws PulsarAdminException, PulsarClientException { + conf.setAllowAutoTopicCreation(allowAutoTopicCreation); + + String partitionedTopic = TopicName.get(domain.value(), "public", "default", + "testCreateProducerWhenSinglePartitionIsDeleted-" + allowAutoTopicCreation) + .toString(); + admin.topics().createPartitionedTopic(partitionedTopic, 3); + admin.topics().delete(TopicName.get(partitionedTopic).getPartition(1).toString()); + + // Non-persistent topic only have the metadata, and no partition, so it works fine. + if (allowAutoTopicCreation || domain == TopicDomain.non_persistent) { + @Cleanup + Producer ignored = pulsarClient.newProducer().topic(partitionedTopic).create(); + } else { + assertThrows(PulsarClientException.TopicDoesNotExistException.class, () -> { + @Cleanup + Producer ignored = pulsarClient.newProducer().topic(partitionedTopic).create(); + }); + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/LookupServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/LookupServiceTest.java index 59cb7ae03d0e3..c4ef53b292b6d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/LookupServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/LookupServiceTest.java @@ -19,13 +19,19 @@ package org.apache.pulsar.client.impl; import static org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace.Mode; +import static org.assertj.core.api.Assertions.assertThat; +import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; import java.util.Collection; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.ProducerConsumerBase; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.testng.annotations.AfterClass; @@ -125,4 +131,80 @@ public void testGetTopicsOfGetTopicsResult(boolean isUsingHttpLookup) throws Exc admin.topics().delete(nonPartitionedTopic, false); } + @Test(dataProvider = "isUsingHttpLookup") + public void testGetPartitionedTopicMetadataByPulsarClient(boolean isUsingHttpLookup) throws PulsarAdminException { + LookupService lookupService = getLookupService(isUsingHttpLookup); + + // metadataAutoCreationEnabled is true. + assertThat(lookupService.getPartitionedTopicMetadata( + TopicName.get(BrokerTestUtil.newUniqueName("persistent://public/default/tp")), true)) + .succeedsWithin(3, TimeUnit.SECONDS) + .matches(n -> n.partitions == 0); + + // metadataAutoCreationEnabled is true. + // Allow the get the metadata of single partition topic, because the auto-creation is enabled. + // But the producer/consumer is unavailable because the topic doesn't have the metadata. + assertThat(lookupService.getPartitionedTopicMetadata( + TopicName.get(BrokerTestUtil.newUniqueName("persistent://public/default/tp") + "-partition-10"), + true)) + .succeedsWithin(3, TimeUnit.SECONDS) + .matches(n -> n.partitions == 0); + + Class expectedExceptionClass = + isUsingHttpLookup ? PulsarClientException.NotFoundException.class : + PulsarClientException.TopicDoesNotExistException.class; + // metadataAutoCreationEnabled is false. + assertThat(lookupService.getPartitionedTopicMetadata( + TopicName.get(BrokerTestUtil.newUniqueName("persistent://public/default/tp")), false)) + .failsWithin(3, TimeUnit.SECONDS) + .withThrowableThat() + .withCauseInstanceOf(expectedExceptionClass); + + // metadataAutoCreationEnabled is false. + assertThat(lookupService.getPartitionedTopicMetadata( + TopicName.get(BrokerTestUtil.newUniqueName("persistent://public/default/tp") + "-partition-10"), + false)) + .failsWithin(3, TimeUnit.SECONDS) + .withThrowableThat() + .withCauseInstanceOf(expectedExceptionClass); + + // Verify the topic exists, and the metadataAutoCreationEnabled is false. + String nonPartitionedTopic = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createNonPartitionedTopic(nonPartitionedTopic); + assertThat(lookupService.getPartitionedTopicMetadata(TopicName.get(nonPartitionedTopic), false)) + .succeedsWithin(3, TimeUnit.SECONDS) + .matches(n -> n.partitions == 0); + + String partitionedTopic = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + String partitionedTopicWithPartitionIndex = partitionedTopic + "-partition-10"; + admin.topics().createPartitionedTopic(partitionedTopic, 20); + assertThat(lookupService.getPartitionedTopicMetadata(TopicName.get(partitionedTopic), false)) + .succeedsWithin(3, TimeUnit.SECONDS) + .matches(n -> n.partitions == 20); + assertThat(lookupService.getPartitionedTopicMetadata(TopicName.get(partitionedTopicWithPartitionIndex), false)) + .succeedsWithin(3, TimeUnit.SECONDS) + .matches(n -> n.partitions == 0); + } + + @Test + public void testGetPartitionedTopicMedataByAdmin() throws PulsarAdminException { + String nonPartitionedTopic = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + String partitionedTopic = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + String partitionedTopicWithPartitionIndex = partitionedTopic + "-partition-10"; + // No topic, so throw the NotFound. + // BTW: The admin api doesn't allow to creat the metadata of topic default. + assertThrows(PulsarAdminException.NotFoundException.class, () -> admin.topics() + .getPartitionedTopicMetadata(nonPartitionedTopic)); + assertThrows(PulsarAdminException.NotFoundException.class, () -> admin.topics() + .getPartitionedTopicMetadata(partitionedTopic)); + assertThrows(PulsarAdminException.NotFoundException.class, + () -> admin.topics().getPartitionedTopicMetadata(partitionedTopicWithPartitionIndex)); + + admin.topics().createNonPartitionedTopic(nonPartitionedTopic); + assertEquals(admin.topics().getPartitionedTopicMetadata(nonPartitionedTopic).partitions, 0); + + admin.topics().createPartitionedTopic(partitionedTopic, 20); + assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopic).partitions, 20); + assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicWithPartitionIndex).partitions, 0); + } } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClientException.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClientException.java index bf32014af7a68..107064f63192d 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClientException.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClientException.java @@ -972,7 +972,9 @@ public TransactionHasOperationFailedException(String msg) { public static Throwable wrap(Throwable t, String msg) { msg += "\n" + t.getMessage(); // wrap an exception with new message info - if (t instanceof TimeoutException) { + if (t instanceof TopicDoesNotExistException) { + return new TopicDoesNotExistException(msg); + } else if (t instanceof TimeoutException) { return new TimeoutException(msg); } else if (t instanceof InvalidConfigurationException) { return new InvalidConfigurationException(msg); From 38b757ead1f5cfcf6a1f2258bafdd76a5e2484ff Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Tue, 8 Apr 2025 17:53:29 +0800 Subject: [PATCH 02/16] [fix][broker] Directly query single topic existence when the topic is partitioned (#24154) Signed-off-by: Zixuan Liu (cherry picked from commit 0d6c6f4b9e88a4cd10df50e6f1a9422772ca0779) --- .../broker/namespace/NamespaceService.java | 16 ++- .../namespace/NamespaceServiceTest.java | 107 +++++++++++++++--- .../client/api/ConsumerCreationTest.java | 2 +- .../client/api/ProducerCreationTest.java | 2 +- .../client/api/PulsarClientException.java | 4 +- 5 files changed, 105 insertions(+), 26 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 04b6794d94f29..145758c73a8c3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -1401,18 +1401,22 @@ public CompletableFuture checkTopicExists(TopicName topic) { return pulsar.getBrokerService().fetchPartitionedTopicMetadataAsync( topic.isPartitioned() ? TopicName.get(topic.getPartitionedTopicName()) : topic) .thenCompose(metadata -> { + // When the topic has metadata: + // - The topic name is non-partitioned, which means that the topic exists. + // - The topic name is partitioned, please check the specific partition. if (metadata.partitions > 0) { if (!topic.isPartitioned()) { return CompletableFuture.completedFuture( TopicExistsInfo.newPartitionedTopicExists(metadata.partitions)); - } else { - if (topic.getPartitionIndex() < metadata.partitions) { - return CompletableFuture.completedFuture( - TopicExistsInfo.newNonPartitionedTopicExists()); - } + } + if (!topic.isPersistent()) { + // A non-persistent partitioned topic contains only metadata. + // Since no actual partitions are created, there's no need to check under /managed-ledgers. + return CompletableFuture.completedFuture(topic.getPartitionIndex() < metadata.partitions + ? TopicExistsInfo.newNonPartitionedTopicExists() + : TopicExistsInfo.newTopicNotExists()); } } - // Direct query the single topic. return checkNonPartitionedTopicExists(topic).thenApply( b -> b ? TopicExistsInfo.newNonPartitionedTopicExists() : TopicExistsInfo.newTopicNotExists()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java index 6fcebff2b81a7..3483ce3809967 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.namespace; import static org.apache.pulsar.broker.resources.LoadBalanceResources.BUNDLE_DATA_BASE_PATH; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; @@ -86,6 +87,7 @@ import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.metadata.api.GetResult; import org.apache.pulsar.metadata.api.MetadataCache; @@ -835,23 +837,6 @@ public Object[] topicDomain() { }; } - @Test(dataProvider = "topicDomain") - public void testCheckTopicExists(String topicDomain) throws Exception { - String topic = topicDomain + "://prop/ns-abc/" + UUID.randomUUID(); - admin.topics().createNonPartitionedTopic(topic); - Awaitility.await().untilAsserted(() -> { - assertTrue(pulsar.getNamespaceService().checkTopicExists(TopicName.get(topic)).get().isExists()); - }); - - String partitionedTopic = topicDomain + "://prop/ns-abc/" + UUID.randomUUID(); - admin.topics().createPartitionedTopic(partitionedTopic, 5); - Awaitility.await().untilAsserted(() -> { - assertTrue(pulsar.getNamespaceService().checkTopicExists(TopicName.get(partitionedTopic)).get().isExists()); - assertTrue(pulsar.getNamespaceService() - .checkTopicExists(TopicName.get(partitionedTopic + "-partition-2")).get().isExists()); - }); - } - @Test public void testAllowedClustersAtNamespaceLevelShouldBeIncludedInAllowedClustersAtTenantLevel() throws Exception { // 1. Setup @@ -978,6 +963,94 @@ public void testNewAllowedClusterAdminAPIAndItsImpactOnReplicationClusterAPI() t pulsar.getConfiguration().setForceDeleteTenantAllowed(false); } + + @Test(dataProvider = "topicDomain") + public void checkTopicExistsForNonPartitionedTopic(String topicDomain) throws Exception { + TopicName topicName = TopicName.get(topicDomain, "prop", "ns-abc", "topic-" + UUID.randomUUID()); + admin.topics().createNonPartitionedTopic(topicName.toString()); + CompletableFuture result = pulsar.getNamespaceService().checkTopicExists(topicName); + assertThat(result) + .succeedsWithin(3, TimeUnit.SECONDS) + .satisfies(n -> { + assertTrue(n.isExists()); + assertEquals(n.getPartitions(), 0); + assertEquals(n.getTopicType(), TopicType.NON_PARTITIONED); + n.recycle(); + }); + } + + @Test(dataProvider = "topicDomain") + public void checkTopicExistsForPartitionedTopic(String topicDomain) throws Exception { + TopicName topicName = TopicName.get(topicDomain, "prop", "ns-abc", "topic-" + UUID.randomUUID()); + admin.topics().createPartitionedTopic(topicName.toString(), 3); + + // Check the topic exists by the partitions. + CompletableFuture result = pulsar.getNamespaceService().checkTopicExists(topicName); + assertThat(result) + .succeedsWithin(3, TimeUnit.SECONDS) + .satisfies(n -> { + assertTrue(n.isExists()); + assertEquals(n.getPartitions(), 3); + assertEquals(n.getTopicType(), TopicType.PARTITIONED); + n.recycle(); + }); + + // Check the specific partition. + result = pulsar.getNamespaceService().checkTopicExists(topicName.getPartition(2)); + assertThat(result) + .succeedsWithin(3, TimeUnit.SECONDS) + .satisfies(n -> { + assertTrue(n.isExists()); + assertEquals(n.getPartitions(), 0); + assertEquals(n.getTopicType(), TopicType.NON_PARTITIONED); + n.recycle(); + }); + + // Partition index is out of range. + result = pulsar.getNamespaceService().checkTopicExists(topicName.getPartition(10)); + assertThat(result) + .succeedsWithin(3, TimeUnit.SECONDS) + .satisfies(n -> { + assertFalse(n.isExists()); + assertEquals(n.getPartitions(), 0); + assertEquals(n.getTopicType(), TopicType.NON_PARTITIONED); + n.recycle(); + }); + } + + @Test(dataProvider = "topicDomain") + public void checkTopicExistsForNonExistentNonPartitionedTopic(String topicDomain) { + TopicName topicName = TopicName.get(topicDomain, "prop", "ns-abc", "topic-" + UUID.randomUUID()); + CompletableFuture result = pulsar.getNamespaceService().checkTopicExists(topicName); + assertThat(result) + .succeedsWithin(3, TimeUnit.SECONDS) + .satisfies(n -> { + // when using the pulsar client to check non_persistent topic, always return true, so ignore to + // check that. + if (topicDomain.equals(TopicDomain.persistent)) { + assertFalse(n.isExists()); + } + n.recycle(); + }); + } + + @Test(dataProvider = "topicDomain") + public void checkTopicExistsForNonExistentPartitionTopic(String topicDomain) { + TopicName topicName = + TopicName.get(topicDomain, "prop", "ns-abc", "topic-" + UUID.randomUUID() + "-partition-10"); + CompletableFuture result = pulsar.getNamespaceService().checkTopicExists(topicName); + assertThat(result) + .succeedsWithin(3, TimeUnit.SECONDS) + .satisfies(n -> { + // when using the pulsar client to check non_persistent topic, always return true, so ignore to + // check that. + if (topicDomain.equals(TopicDomain.persistent)) { + assertFalse(n.isExists()); + } + n.recycle(); + }); + } + /** * 1. Manually trigger "LoadReportUpdaterTask" * 2. Registry another new zk-node-listener "waitForBrokerChangeNotice". diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerCreationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerCreationTest.java index a81dbe02b3447..195485739e0d6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerCreationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerCreationTest.java @@ -117,7 +117,7 @@ private void testCreateConsumerWhenSinglePartitionIsDeleted(TopicDomain domain, Consumer ignored = pulsarClient.newConsumer().topic(partitionedTopic).subscriptionName("my-sub").subscribe(); } else { - assertThrows(PulsarClientException.TopicDoesNotExistException.class, () -> { + assertThrows(PulsarClientException.NotFoundException.class, () -> { @Cleanup Consumer ignored = pulsarClient.newConsumer().topic(partitionedTopic).subscriptionName("my-sub").subscribe(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerCreationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerCreationTest.java index dee53a5df01f1..e13423a213119 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerCreationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerCreationTest.java @@ -261,7 +261,7 @@ private void testCreateProducerWhenSinglePartitionIsDeleted(TopicDomain domain, @Cleanup Producer ignored = pulsarClient.newProducer().topic(partitionedTopic).create(); } else { - assertThrows(PulsarClientException.TopicDoesNotExistException.class, () -> { + assertThrows(PulsarClientException.NotFoundException.class, () -> { @Cleanup Producer ignored = pulsarClient.newProducer().topic(partitionedTopic).create(); }); diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClientException.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClientException.java index 107064f63192d..5005549f476ee 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClientException.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClientException.java @@ -972,7 +972,9 @@ public TransactionHasOperationFailedException(String msg) { public static Throwable wrap(Throwable t, String msg) { msg += "\n" + t.getMessage(); // wrap an exception with new message info - if (t instanceof TopicDoesNotExistException) { + if (t instanceof NotFoundException) { + return new NotFoundException(msg); + } else if (t instanceof TopicDoesNotExistException) { return new TopicDoesNotExistException(msg); } else if (t instanceof TimeoutException) { return new TimeoutException(msg); From 359e7ae5953dba9bf6338c0c330fb4dea3f45a85 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Wed, 14 May 2025 10:56:23 +0800 Subject: [PATCH 03/16] [fix][broker] Allow recreation of partitioned topic after metadata loss (#24225) Signed-off-by: Zixuan Liu (cherry picked from commit 2dd1ef6a76fcc77efe482ae663e5a1f5b44aa796) --- .../pulsar/broker/admin/AdminResource.java | 85 ++++++++++++------- .../admin/impl/PersistentTopicsBase.java | 5 +- .../broker/namespace/NamespaceService.java | 4 + .../pulsar/broker/admin/AdminApiTest.java | 19 +++++ .../apache/pulsar/broker/admin/AdminTest.java | 10 +-- 5 files changed, 82 insertions(+), 41 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java index d35ea51d2835b..bfa1fdc812b7b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java @@ -44,6 +44,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authorization.AuthorizationService; +import org.apache.pulsar.broker.namespace.TopicExistsInfo; import org.apache.pulsar.broker.resources.ClusterResources; import org.apache.pulsar.broker.service.TopicEventsListener.EventStage; import org.apache.pulsar.broker.service.TopicEventsListener.TopicEvent; @@ -54,7 +55,6 @@ import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.admin.internal.TopicsImpl; -import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace; import org.apache.pulsar.common.naming.Constants; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceName; @@ -74,6 +74,7 @@ import org.apache.pulsar.common.policies.data.SubscribeRate; import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.policies.data.TopicPolicies; +import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.common.policies.data.impl.AutoSubscriptionCreationOverrideImpl; import org.apache.pulsar.common.policies.data.impl.DispatchRateImpl; import org.apache.pulsar.common.util.Codec; @@ -614,12 +615,32 @@ protected void internalCreatePartitionedTopic(AsyncResponse asyncResponse, int n return CompletableFuture.completedFuture(null); }) .thenCompose(__ -> checkTopicExistsAsync(topicName)) - .thenAccept(exists -> { - if (exists) { - log.warn("[{}] Failed to create already existing topic {}", clientAppId(), topicName); - throw new RestException(Status.CONFLICT, "This topic already exists"); + .thenAccept(topicExistsInfo -> { + try { + if (topicExistsInfo.isExists()) { + if (topicExistsInfo.getTopicType().equals(TopicType.NON_PARTITIONED) + || (topicExistsInfo.getTopicType().equals(TopicType.PARTITIONED) + && !createLocalTopicOnly)) { + log.warn("[{}] Failed to create already existing topic {}", clientAppId(), topicName); + throw new RestException(Status.CONFLICT, "This topic already exists"); + } + } + } finally { + topicExistsInfo.recycle(); } }) + .thenCompose(__ -> getMaxPartitionIndex(topicName) + .thenAccept(existingMaxPartitionIndex -> { + // Case 1: Metadata loss — user tries to recreate the partitioned topic. + // Case 2: Non-partitioned topic — user attempts to convert it to a partitioned topic. + if (existingMaxPartitionIndex >= numPartitions) { + int requiredMinPartitions = existingMaxPartitionIndex + 1; + throw new RestException(Status.CONFLICT, String.format( + "The topic has a max partition index of %d, the number of partitions must be " + + "at least %d", + existingMaxPartitionIndex, requiredMinPartitions)); + } + })) .thenRun(() -> { for (int i = 0; i < numPartitions; i++) { pulsar().getBrokerService().getTopicEventsDispatcher() @@ -653,6 +674,26 @@ && pulsar().getConfig().isCreateTopicToRemoteClusterForReplication()) { }); } + private CompletableFuture getMaxPartitionIndex(TopicName topicName) { + if (topicName.getDomain() == TopicDomain.persistent) { + return getPulsarResources().getTopicResources().listPersistentTopicsAsync(topicName.getNamespaceObject()) + .thenApply(list -> { + int maxIndex = -1; + for (String s : list) { + TopicName item = TopicName.get(s); + if (item.isPartitioned() && item.getPartitionedTopicName() + .equals(topicName.getPartitionedTopicName())) { + if (item.getPartitionIndex() > maxIndex) { + maxIndex = item.getPartitionIndex(); + } + } + } + return maxIndex; + }); + } + return CompletableFuture.completedFuture(-1); + } + private void internalCreatePartitionedTopicToReplicatedClustersInBackground(int numPartitions) { getNamespaceReplicatedClustersAsync(namespaceName) .thenAccept(clusters -> { @@ -743,36 +784,16 @@ protected Map> internalCreatePartitionedTopicToR /** * Check the exists topics contains the given topic. - * Since there are topic partitions and non-partitioned topics in Pulsar, must ensure both partitions - * and non-partitioned topics are not duplicated. So, if compare with a partition name, we should compare - * to the partitioned name of this partition. + * Since there are topic partitions and non-partitioned topics in Pulsar. This check ensures that + * there is no duplication between them. Partition *instances* (e.g., topic-partition-0) + * are not considered individually — only the base partitioned topic name is checked. + * + *

Note: Be sure to recycle the {@link TopicExistsInfo} after it is no longer needed.

* * @param topicName given topic name */ - protected CompletableFuture checkTopicExistsAsync(TopicName topicName) { - return pulsar().getNamespaceService().getListOfTopics(topicName.getNamespaceObject(), - CommandGetTopicsOfNamespace.Mode.ALL) - .thenCompose(topics -> { - boolean persistentResourceCreated = false; - for (String topic : topics) { - if (topicName.getPartitionedTopicName().equals( - TopicName.get(topic).getPartitionedTopicName())) { - persistentResourceCreated = true; - break; - } - } - if (persistentResourceCreated) { - return CompletableFuture.completedFuture(true); - } - return pulsar().getPulsarResources().getNamespaceResources().getPartitionedTopicResources() - .getPartitionedTopicMetadataAsync(TopicName.get(topicName.getPartitionedTopicName()), false) - .thenApply(optMetadata -> { - if (optMetadata.isEmpty()) { - return false; - } - return optMetadata.get().partitions > topicName.getPartitionIndex(); - }); - }); + protected CompletableFuture checkTopicExistsAsync(TopicName topicName) { + return pulsar().getNamespaceService().checkTopicExistsAsync(topicName); } private CompletableFuture provisionPartitionedTopicPath(int numPartitions, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 0a683f4b8729d..3850131a8d1c5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -3711,8 +3711,9 @@ protected CompletableFuture preValidation(boolean authoritative) { } return ret .thenCompose(__ -> checkTopicExistsAsync(topicName)) - .thenCompose(exist -> { - if (!exist) { + .thenCompose(topicExistsInfo -> { + if (!topicExistsInfo.isExists()) { + topicExistsInfo.recycle(); throw new RestException(Status.NOT_FOUND, getTopicNotFoundErrorMessage(topicName.toString())); } else { return getPartitionedTopicMetadataAsync(topicName, false, false) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 145758c73a8c3..a92acf2177e9c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -1397,6 +1397,10 @@ public CompletableFuture checkTopicExistsAsync(TopicName topic) */ @Deprecated public CompletableFuture checkTopicExists(TopicName topic) { + // Exclude the heartbeat topic. + if (isHeartbeatNamespace(topic)) { + return CompletableFuture.completedFuture(TopicExistsInfo.newNonPartitionedTopicExists()); + } // For non-persistent/persistent partitioned topic, which has metadata. return pulsar.getBrokerService().fetchPartitionedTopicMetadataAsync( topic.isPartitioned() ? TopicName.get(topic.getPartitionedTopicName()) : topic) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java index aa937fb31fb67..8dabc75e52d8b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java @@ -52,6 +52,7 @@ import java.util.TreeSet; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -3939,4 +3940,22 @@ public void testPermissionsAllowAclChangesOnNonExistentTopics() { pulsar.getConfiguration().setAllowAclChangesOnNonExistentTopics(false); } } + + @Test + public void testRecreatePartitionedTopicAfterMetadataLoss() + throws PulsarAdminException, ExecutionException, InterruptedException { + String namespace = "prop-xyz/ns1/"; + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://" + namespace + random; + admin.topics().createPartitionedTopic(topic, 5); + + // Delete the topic metadata. + pulsar.getPulsarResources().getNamespaceResources().getPartitionedTopicResources() + .deletePartitionedTopicAsync(TopicName.get(topic)).get(); + List partitionedTopicList = admin.topics().getPartitionedTopicList(namespace); + assertThat(partitionedTopicList).doesNotContain(topic); + + // Create the partitioned topic again. + admin.topics().createPartitionedTopic(topic, 5); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java index dbff93319a099..b80605f8f49a3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.admin; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -69,7 +70,6 @@ import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.web.PulsarWebResource; import org.apache.pulsar.broker.web.RestException; -import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace; import org.apache.pulsar.common.conf.InternalConfigurationData; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; @@ -953,15 +953,11 @@ public void test500Error() throws Exception { final String partitionedTopicName = "error-500-topic"; AsyncResponse response1 = mock(AsyncResponse.class); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(RestException.class); - NamespaceName namespaceName = NamespaceName.get(property, cluster, namespace); CompletableFuture> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("500 error contains error message")); NamespaceService namespaceService = pulsar.getNamespaceService(); - - doReturn(future).when(namespaceService).getListOfTopics(namespaceName, CommandGetTopicsOfNamespace.Mode.ALL); - persistentTopics.createPartitionedTopic(response1, property, cluster, namespace, - partitionedTopicName, 5, false); - + doReturn(future).when(namespaceService).checkTopicExists(any()); + persistentTopics.createPartitionedTopic(response1, property, cluster, namespace, partitionedTopicName, 5, false); verify(response1, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode()); From cc1ba9051240fc38d82775d3ccce462b48c91b3e Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Thu, 19 Mar 2026 12:01:19 +0800 Subject: [PATCH 04/16] [feat][broker] PIP-398: Subscription replication on the namespace and topic levels (#29) # Conflicts: # pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/TopicPolicies.java # pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicPoliciesImpl.java # pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopicPolicies.java # pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java # pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java --- .../broker/admin/impl/NamespacesBase.java | 10 + .../admin/impl/PersistentTopicsBase.java | 23 ++- .../pulsar/broker/admin/v2/Namespaces.java | 37 ++++ .../broker/admin/v2/PersistentTopics.java | 79 ++++++++ .../pulsar/broker/service/AbstractTopic.java | 3 + .../persistent/PersistentSubscription.java | 64 ++++--- .../service/persistent/PersistentTopic.java | 27 ++- ...dminApiReplicateSubscriptionStateTest.java | 155 ++++++++++++++++ .../PersistentSubscriptionTest.java | 29 +++ ...ReplicatedSubscriptionsIsDisabledTest.java | 7 +- .../client/api/ReplicateSubscriptionTest.java | 174 +++++++++++++++++- .../pulsar/client/admin/Namespaces.java | 49 +++++ .../pulsar/client/admin/TopicPolicies.java | 105 +++++++++++ .../pulsar/common/policies/data/Policies.java | 9 +- .../client/admin/internal/NamespacesImpl.java | 22 +++ .../admin/internal/TopicPoliciesImpl.java | 61 ++++++ .../pulsar/admin/cli/CmdNamespaces.java | 45 +++++ .../pulsar/admin/cli/CmdTopicPolicies.java | 118 ++++++++++++ .../policies/data/HierarchyTopicPolicies.java | 3 + .../common/policies/data/PolicyName.java | 2 + .../common/policies/data/TopicPolicies.java | 2 + .../cli/ReplicateSubscriptionStateTest.java | 159 ++++++++++++++++ .../src/test/resources/pulsar-cli.xml | 1 + 23 files changed, 1134 insertions(+), 50 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiReplicateSubscriptionStateTest.java create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/ReplicateSubscriptionStateTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index f28b07d8a9a03..024dc30e0ad1c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -2721,6 +2721,16 @@ protected void internalScanOffloadedLedgers(OffloaderObjectsScannerUtils.Scanner } + protected CompletableFuture internalSetReplicateSubscriptionStateAsync(Boolean enabled) { + return validatePoliciesReadOnlyAccessAsync() + .thenCompose(__ -> validateNamespacePolicyOperationAsync(namespaceName, + PolicyName.REPLICATED_SUBSCRIPTION, PolicyOperation.WRITE)) + .thenCompose(__ -> updatePoliciesAsync(namespaceName, policies -> { + policies.replicate_subscription_state = enabled; + return policies; + })); + } + protected CompletableFuture internalSetEntryFiltersPerTopicAsync(EntryFilters entryFilters) { return validateNamespacePolicyOperationAsync(namespaceName, PolicyName.ENTRY_FILTERS, PolicyOperation.WRITE) .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 3850131a8d1c5..94bab3e41b1fc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -4921,8 +4921,27 @@ protected CompletableFuture internalTruncateTopicAsync(boolean authoritati } } + protected CompletableFuture internalSetEnableReplicatedSubscription(Boolean enabled, + boolean isGlobal) { + return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, isGlobal, false, policies -> { + policies.setReplicateSubscriptionState(enabled); + }); + } + + protected CompletableFuture internalGetReplicateSubscriptionState(boolean applied, + boolean isGlobal) { + return getTopicPoliciesAsyncWithRetry(topicName, isGlobal) + .thenApply(op -> op.map(TopicPolicies::getReplicateSubscriptionState) + .orElseGet(() -> { + if (applied) { + return getNamespacePolicies(namespaceName).replicate_subscription_state; + } + return null; + })); + } + protected void internalSetReplicatedSubscriptionStatus(AsyncResponse asyncResponse, String subName, - boolean authoritative, boolean enabled) { + boolean authoritative, Boolean enabled) { log.info("[{}] Attempting to change replicated subscription status to {} - {} {}", clientAppId(), enabled, topicName, subName); @@ -5015,7 +5034,7 @@ protected void internalSetReplicatedSubscriptionStatus(AsyncResponse asyncRespon } private void internalSetReplicatedSubscriptionStatusForNonPartitionedTopic( - AsyncResponse asyncResponse, String subName, boolean authoritative, boolean enabled) { + AsyncResponse asyncResponse, String subName, boolean authoritative, Boolean enabled) { // Redirect the request to the appropriate broker if this broker is not the owner of the topic validateTopicOwnershipAsync(topicName, authoritative) .thenCompose(__ -> getTopicReferenceAsync(topicName)) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index 5058eccc4011e..f172e36ff6d5a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -3169,5 +3169,42 @@ public void getNamespaceAllowedClusters(@Suspended AsyncResponse asyncResponse, }); } + @GET + @Path("/{tenant}/{namespace}/replicateSubscriptionState") + @ApiOperation(value = "Get the enabled status of subscription replication on a namespace.", response = + Boolean.class) + @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist")}) + public Boolean getReplicateSubscriptionState(@PathParam("tenant") String tenant, + @PathParam("namespace") String namespace) { + validateNamespaceName(tenant, namespace); + validateNamespacePolicyOperation(NamespaceName.get(tenant, namespace), + PolicyName.REPLICATED_SUBSCRIPTION, PolicyOperation.READ); + + Policies policies = getNamespacePolicies(namespaceName); + return policies.replicate_subscription_state; + } + + @POST + @Path("/{tenant}/{namespace}/replicateSubscriptionState") + @ApiOperation(value = "Enable or disable subscription replication on a namespace.") + @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist")}) + public void setReplicateSubscriptionState(@Suspended final AsyncResponse asyncResponse, + @PathParam("tenant") String tenant, + @PathParam("namespace") String namespace, + @ApiParam(value = "Whether to enable subscription replication", + required = true) + Boolean enabled) { + validateNamespaceName(tenant, namespace); + internalSetReplicateSubscriptionStateAsync(enabled) + .thenRun(() -> asyncResponse.resume(Response.noContent().build())) + .exceptionally(ex -> { + log.error("set replicate subscription state failed", ex); + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); + } + private static final Logger log = LoggerFactory.getLogger(Namespaces.class); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java index bb8eb6393cec7..3b854e49f47cb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java @@ -4648,6 +4648,85 @@ public void getReplicatedSubscriptionStatus( internalGetReplicatedSubscriptionStatus(asyncResponse, decode(encodedSubName), authoritative); } + @POST + @Path("/{tenant}/{namespace}/{topic}/replicateSubscriptionState") + @ApiOperation(value = "Enable or disable subscription replication on a topic.") + @ApiResponses(value = { + @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), + @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or " + + "subscriber is not authorized to access this operation"), + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Topic or subscription does not exist"), + @ApiResponse(code = 405, message = "Operation not allowed on this topic"), + @ApiResponse(code = 412, message = "Can't find owner for topic"), + @ApiResponse(code = 500, message = "Internal server error"), + @ApiResponse(code = 503, message = "Failed to validate global cluster configuration")}) + public void setReplicateSubscriptionState( + @Suspended final AsyncResponse asyncResponse, + @ApiParam(value = "Specify the tenant", required = true) + @PathParam("tenant") String tenant, + @ApiParam(value = "Specify the namespace", required = true) + @PathParam("namespace") String namespace, + @ApiParam(value = "Specify topic name", required = true) + @PathParam("topic") @Encoded String encodedTopic, + @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal, + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, + @ApiParam(value = "Whether to enable subscription replication", required = true) + Boolean enabled) { + validateTopicName(tenant, namespace, encodedTopic); + validateTopicOperationAsync(topicName, TopicOperation.SET_REPLICATED_SUBSCRIPTION_STATUS) + .thenCompose(__ -> preValidation(authoritative)) + .thenCompose(__ -> internalSetEnableReplicatedSubscription(enabled, isGlobal)) + .thenRun(() -> { + log.info( + "[{}] Successfully set topic replicated subscription enabled: tenant={}, namespace={}, " + + "topic={}, isGlobal={}", + clientAppId(), + tenant, + namespace, + topicName.getLocalName(), + isGlobal); + asyncResponse.resume(Response.noContent().build()); + }) + .exceptionally(ex -> { + handleTopicPolicyException("setReplicateSubscriptionState", ex, asyncResponse); + return null; + }); + } + + @GET + @Path("/{tenant}/{namespace}/{topic}/replicateSubscriptionState") + @ApiOperation(value = "Get the enabled status of subscription replication on a topic.") + @ApiResponses(value = { + @ApiResponse(code = 401, message = "Don't have permission to administrate resources"), + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Topic does not exist"), + @ApiResponse(code = 412, message = "Can't find owner for topic"), + @ApiResponse(code = 500, message = "Internal server error")}) + public void getReplicateSubscriptionState( + @Suspended AsyncResponse asyncResponse, + @ApiParam(value = "Specify the tenant", required = true) + @PathParam("tenant") String tenant, + @ApiParam(value = "Specify the namespace", required = true) + @PathParam("namespace") String namespace, + @ApiParam(value = "Specify topic name", required = true) + @PathParam("topic") @Encoded String encodedTopic, + @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal, + @QueryParam("applied") @DefaultValue("false") boolean applied, + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + validateTopicName(tenant, namespace, encodedTopic); + validateTopicOperationAsync(topicName, TopicOperation.GET_REPLICATED_SUBSCRIPTION_STATUS) + .thenCompose(__ -> preValidation(authoritative)) + .thenCompose(__ -> internalGetReplicateSubscriptionState(applied, isGlobal)) + .thenAccept(asyncResponse::resume) + .exceptionally(ex -> { + handleTopicPolicyException("getReplicateSubscriptionState", ex, asyncResponse); + return null; + }); + } + @GET @Path("/{tenant}/{namespace}/{topic}/schemaCompatibilityStrategy") @ApiOperation(value = "Get schema compatibility strategy on a topic", response = SchemaCompatibilityStrategy.class) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 8af4a3264d97e..e0b310712d13f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -251,6 +251,7 @@ protected void updateTopicPolicy(TopicPolicies data) { topicPolicies.getSchemaCompatibilityStrategy() .updateTopicValue(formatSchemaCompatibilityStrategy(data.getSchemaCompatibilityStrategy()), isGlobalPolicies); + topicPolicies.getReplicateSubscriptionState().updateTopicValue(data.getReplicateSubscriptionState()); } topicPolicies.getRetentionPolicies().updateTopicValue(data.getRetentionPolicies(), isGlobalPolicies); topicPolicies.getDispatcherPauseOnAckStatePersistentEnabled() @@ -314,6 +315,8 @@ protected void updateTopicPolicyByNamespacePolicy(Policies namespacePolicies) { if (!isSystemTopic()) { updateNamespacePublishRate(namespacePolicies, brokerService.getPulsar().getConfig().getClusterName()); updateNamespaceDispatchRate(namespacePolicies, brokerService.getPulsar().getConfig().getClusterName()); + topicPolicies.getReplicateSubscriptionState() + .updateNamespaceValue(namespacePolicies.replicate_subscription_state); } topicPolicies.getRetentionPolicies().updateNamespaceValue(namespacePolicies.retention_policies); topicPolicies.getCompactionThreshold().updateNamespaceValue(namespacePolicies.compaction_threshold); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index d23db646963f4..f0103c4d13fc6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -118,14 +118,13 @@ public class PersistentSubscription extends AbstractSubscription { // for connected subscriptions, message expiry will be checked if the backlog is greater than this threshold private static final int MINIMUM_BACKLOG_FOR_EXPIRY_CHECK = 1000; - private static final String REPLICATED_SUBSCRIPTION_PROPERTY = "pulsar.replicated.subscription"; + static final String REPLICATED_SUBSCRIPTION_PROPERTY = "pulsar.replicated.subscription"; // Map of properties that is used to mark this subscription as "replicated". // Since this is the only field at this point, we can just keep a static // instance of the map. - private static final Map REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES = - Map.of(REPLICATED_SUBSCRIPTION_PROPERTY, 1L); - private static final Map NON_REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES = Map.of(); + private static final Map REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES = new TreeMap<>(); + private static final Map NON_REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES = Collections.emptyMap(); private volatile ReplicatedSubscriptionSnapshotCache replicatedSubscriptionSnapshotCache; @Getter @@ -133,16 +132,21 @@ public class PersistentSubscription extends AbstractSubscription { private volatile Map subscriptionProperties; private volatile CompletableFuture fenceFuture; private volatile CompletableFuture inProgressResetCursorFuture; - private volatile Boolean replicatedControlled; + private final ServiceConfiguration config; + static { + REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES.put(REPLICATED_SUBSCRIPTION_PROPERTY, 1L); + } + static Map getBaseCursorProperties(Boolean isReplicated) { return isReplicated != null && isReplicated ? REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES : NON_REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES; } - static boolean isCursorFromReplicatedSubscription(ManagedCursor cursor) { - return cursor.getProperties().containsKey(REPLICATED_SUBSCRIPTION_PROPERTY); + public static boolean isCursorFromReplicatedSubscription(ManagedCursor cursor) { + Long v = cursor.getProperties().get(REPLICATED_SUBSCRIPTION_PROPERTY); + return v != null && v == 1L; } public PersistentSubscription(PersistentTopic topic, String subscriptionName, ManagedCursor cursor, @@ -159,9 +163,7 @@ public PersistentSubscription(PersistentTopic topic, String subscriptionName, Ma this.subName = subscriptionName; this.fullName = MoreObjects.toStringHelper(this).add("topic", topicName).add("name", subName).toString(); this.expiryMonitor = new PersistentMessageExpiryMonitor(topic, subscriptionName, cursor, this); - if (replicated != null) { - this.setReplicated(replicated); - } + this.setReplicated(replicated); this.subscriptionProperties = MapUtils.isEmpty(subscriptionProperties) ? Collections.emptyMap() : Collections.unmodifiableMap(subscriptionProperties); if (config.isTransactionCoordinatorEnabled() @@ -199,10 +201,19 @@ public boolean isReplicated() { return replicatedSubscriptionSnapshotCache != null; } - public boolean setReplicated(boolean replicated) { - replicatedControlled = replicated; + public boolean setReplicated(Boolean replicated) { + return setReplicated(replicated, true); + } + + protected boolean setReplicated(Boolean replicated, boolean isPersistent) { + ServiceConfiguration config = topic.getBrokerService().getPulsar().getConfig(); + if (!config.isEnableReplicatedSubscriptions()) { + log.warn("[{}][{}] Failed set replicated subscription status to {}, please enable the " + + "configuration enableReplicatedSubscriptions", topicName, subName, replicated); + return false; + } - if (!replicated || !config.isEnableReplicatedSubscriptions()) { + if (replicated == null || !replicated) { this.replicatedSubscriptionSnapshotCache = null; } else if (this.replicatedSubscriptionSnapshotCache == null) { this.replicatedSubscriptionSnapshotCache = new ReplicatedSubscriptionSnapshotCache(subName, @@ -210,20 +221,18 @@ public boolean setReplicated(boolean replicated) { getCursor().getManagedLedger()::getNumberOfEntries); } - if (this.cursor != null) { - if (replicated) { - if (!config.isEnableReplicatedSubscriptions()) { - log.warn("[{}][{}] Failed set replicated subscription status to {}, please enable the " - + "configuration enableReplicatedSubscriptions", topicName, subName, replicated); - } else { - return this.cursor.putProperty(REPLICATED_SUBSCRIPTION_PROPERTY, 1L); - } - } else { - return this.cursor.removeProperty(REPLICATED_SUBSCRIPTION_PROPERTY); - } + if (!isPersistent) { + return true; } - return false; + if (this.cursor == null) { + return false; + } + if (replicated != null && replicated) { + return this.cursor.putProperty(REPLICATED_SUBSCRIPTION_PROPERTY, 1L); + } else { + return this.cursor.removeProperty(REPLICATED_SUBSCRIPTION_PROPERTY); + } } @Override @@ -1653,9 +1662,4 @@ public PositionInPendingAckStats checkPositionInPendingAckState(Position positio } private static final Logger log = LoggerFactory.getLogger(PersistentSubscription.class); - - @VisibleForTesting - public Boolean getReplicatedControlled() { - return replicatedControlled; - } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index a97159553a48e..101bf30de96fa 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -939,13 +939,6 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St } return brokerService.checkTopicNsOwnership(getName()).thenCompose(__ -> { - Boolean replicatedSubscriptionState = replicatedSubscriptionStateArg; - if (replicatedSubscriptionState != null && replicatedSubscriptionState - && !brokerService.pulsar().getConfiguration().isEnableReplicatedSubscriptions()) { - log.warn("[{}] Replicated Subscription is disabled by broker.", getName()); - replicatedSubscriptionState = false; - } - if (subType == SubType.Key_Shared && !brokerService.pulsar().getConfiguration().isSubscriptionKeySharedEnable()) { return FutureUtil.failedFuture( @@ -1012,7 +1005,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St CompletableFuture subscriptionFuture = isDurable ? getDurableSubscription(subscriptionName, initialPosition, startMessageRollbackDurationSec, - replicatedSubscriptionState, subscriptionProperties) + replicatedSubscriptionStateArg, subscriptionProperties) : getNonDurableSubscription(subscriptionName, startMessageId, initialPosition, startMessageRollbackDurationSec, readCompacted, subscriptionProperties); @@ -4414,9 +4407,23 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem }); } + private boolean replicateSubscriptionStateEnabledByTopicPolicies; + public synchronized void checkReplicatedSubscriptionControllerState() { + Boolean replicatedSubscriptionState = topicPolicies.getReplicateSubscriptionState().get(); AtomicBoolean shouldBeEnabled = new AtomicBoolean(false); subscriptions.forEach((name, subscription) -> { + if (Boolean.TRUE.equals(replicatedSubscriptionState)) { + if (!subscription.isReplicated() && subscription.setReplicated(true, false)) { + replicateSubscriptionStateEnabledByTopicPolicies = true; + } + } else if (replicateSubscriptionStateEnabledByTopicPolicies) { + if (!PersistentSubscription.isCursorFromReplicatedSubscription(subscription.getCursor())) { + if (subscription.isReplicated()) { + subscription.setReplicated(false, false); + } + } + } if (subscription.isReplicated()) { shouldBeEnabled.set(true); } @@ -4426,6 +4433,10 @@ public synchronized void checkReplicatedSubscriptionControllerState() { if (log.isDebugEnabled()) { log.debug("[{}] There are no replicated subscriptions on the topic", topic); } + // When no replication subscriptions, set the flag to false. + if (replicateSubscriptionStateEnabledByTopicPolicies) { + replicateSubscriptionStateEnabledByTopicPolicies = false; + } } checkReplicatedSubscriptionControllerState(shouldBeEnabled.get()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiReplicateSubscriptionStateTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiReplicateSubscriptionStateTest.java new file mode 100644 index 0000000000000..815b31c5601b0 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiReplicateSubscriptionStateTest.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin; + +import static org.awaitility.Awaitility.await; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Test(groups = "broker-admin") +public class AdminApiReplicateSubscriptionStateTest extends MockedPulsarServiceBaseTest { + @BeforeClass + @Override + public void setup() throws Exception { + super.internalSetup(); + super.setupDefaultTenantAndNamespace(); + } + + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + conf.setTopicLevelPoliciesEnabled(true); + conf.setSystemTopicEnabled(true); + } + + @AfterClass + @Override + public void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testReplicateSubscriptionStateOnNamespaceLevel() throws PulsarAdminException { + String nsName = "public/testReplicateSubscriptionState" + System.nanoTime(); + admin.namespaces().createNamespace(nsName); + assertNull(admin.namespaces().getReplicateSubscriptionState(nsName)); + + String topicName = nsName + "/topic" + System.nanoTime(); + admin.topics().createNonPartitionedTopic(topicName); + assertNull(admin.topicPolicies().getReplicateSubscriptionState(topicName, true)); + + admin.namespaces().setReplicateSubscriptionState(nsName, true); + assertTrue(admin.namespaces().getReplicateSubscriptionState(nsName)); + await().untilAsserted(() -> { + assertNull(admin.topicPolicies().getReplicateSubscriptionState(topicName, false)); + assertTrue(admin.topicPolicies().getReplicateSubscriptionState(topicName, true)); + }); + + admin.namespaces().setReplicateSubscriptionState(nsName, false); + assertFalse(admin.namespaces().getReplicateSubscriptionState(nsName)); + await().untilAsserted(() -> { + assertFalse(admin.topicPolicies().getReplicateSubscriptionState(topicName, true)); + assertNull(admin.topicPolicies().getReplicateSubscriptionState(topicName, false)); + }); + + admin.namespaces().setReplicateSubscriptionState(nsName, null); + assertNull(admin.namespaces().getReplicateSubscriptionState(nsName)); + await().untilAsserted(() -> { + assertNull(admin.topicPolicies().getReplicateSubscriptionState(topicName, true)); + assertNull(admin.topicPolicies().getReplicateSubscriptionState(topicName, false)); + }); + } + + @Test + public void testReplicateSubscriptionStateOnTopicLevel() throws PulsarAdminException { + String nsName = "public/testReplicateSubscriptionState" + System.nanoTime(); + admin.namespaces().createNamespace(nsName); + assertNull(admin.namespaces().getReplicateSubscriptionState(nsName)); + + String topicName = nsName + "/topic" + System.nanoTime(); + admin.topics().createNonPartitionedTopic(topicName); + assertNull(admin.topicPolicies().getReplicateSubscriptionState(topicName, true)); + assertNull(admin.topicPolicies().getReplicateSubscriptionState(topicName, false)); + + admin.topicPolicies().setReplicateSubscriptionState(topicName, true); + await().untilAsserted(() -> { + assertEquals(admin.topicPolicies().getReplicateSubscriptionState(topicName, true), Boolean.TRUE); + assertEquals(admin.topicPolicies().getReplicateSubscriptionState(topicName, false), Boolean.TRUE); + }); + + admin.topicPolicies().setReplicateSubscriptionState(topicName, false); + await().untilAsserted(() -> { + assertEquals(admin.topicPolicies().getReplicateSubscriptionState(topicName, true), Boolean.FALSE); + assertEquals(admin.topicPolicies().getReplicateSubscriptionState(topicName, false), Boolean.FALSE); + }); + + admin.topicPolicies().setReplicateSubscriptionState(topicName, null); + await().untilAsserted(() -> { + assertNull(admin.topicPolicies().getReplicateSubscriptionState(topicName, true)); + assertNull(admin.topicPolicies().getReplicateSubscriptionState(topicName, true)); + }); + } + + @DataProvider + Object[] replicateSubscriptionStatePriorityLevelDataProvider() { + return new Object[]{ + true, + false, + null, + }; + } + + @Test(dataProvider = "replicateSubscriptionStatePriorityLevelDataProvider") + public void testReplicateSubscriptionStatePriorityLevel(Boolean enabledOnNamespace) + throws PulsarAdminException { + String nsName = "public/testReplicateSubscriptionState" + System.nanoTime(); + admin.namespaces().createNamespace(nsName); + assertNull(admin.namespaces().getReplicateSubscriptionState(nsName)); + admin.namespaces().setReplicateSubscriptionState(nsName, enabledOnNamespace); + + String topicName = nsName + "/topic" + System.nanoTime(); + admin.topics().createNonPartitionedTopic(topicName); + + admin.topicPolicies().setReplicateSubscriptionState(topicName, false); + await().untilAsserted(() -> { + assertFalse(admin.topicPolicies().getReplicateSubscriptionState(topicName, true)); + assertFalse(admin.topicPolicies().getReplicateSubscriptionState(topicName, false)); + }); + + admin.topicPolicies().setReplicateSubscriptionState(topicName, true); + await().untilAsserted(() -> { + assertTrue(admin.topicPolicies().getReplicateSubscriptionState(topicName, true)); + assertTrue(admin.topicPolicies().getReplicateSubscriptionState(topicName, false)); + }); + + admin.topicPolicies().setReplicateSubscriptionState(topicName, null); + await().untilAsserted(() -> { + assertNull(admin.topicPolicies().getReplicateSubscriptionState(topicName, false)); + assertEquals(admin.topicPolicies().getReplicateSubscriptionState(topicName, true), enabledOnNamespace); + }); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java index dcd966e403067..3b911d0a87c24 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.service.persistent; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doCallRealMethod; @@ -29,12 +30,15 @@ import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.Position; @@ -274,4 +278,29 @@ public CompletableFuture checkInitializedBefore(PersistentSubscription return CompletableFuture.completedFuture(true); } } + + @Test + public void testGetReplicatedSubscriptionConfiguration() { + Map properties = PersistentSubscription.getBaseCursorProperties(true); + assertThat(properties).containsEntry(PersistentSubscription.REPLICATED_SUBSCRIPTION_PROPERTY, 1L); + ManagedCursor cursor = mock(ManagedCursor.class); + doReturn(properties).when(cursor).getProperties(); + assertThat(PersistentSubscription.isCursorFromReplicatedSubscription(cursor)).isTrue(); + + properties = new HashMap<>(); + properties.put(PersistentSubscription.REPLICATED_SUBSCRIPTION_PROPERTY, 10L); + doReturn(properties).when(cursor).getProperties(); + assertThat(PersistentSubscription.isCursorFromReplicatedSubscription(cursor)).isFalse(); + + properties = new HashMap<>(); + properties.put(PersistentSubscription.REPLICATED_SUBSCRIPTION_PROPERTY, -1L); + doReturn(properties).when(cursor).getProperties(); + assertThat(PersistentSubscription.isCursorFromReplicatedSubscription(cursor)).isFalse(); + + properties = PersistentSubscription.getBaseCursorProperties(false); + assertThat(properties).doesNotContainKey(PersistentSubscription.REPLICATED_SUBSCRIPTION_PROPERTY); + + properties = PersistentSubscription.getBaseCursorProperties(null); + assertThat(properties).doesNotContainKey(PersistentSubscription.REPLICATED_SUBSCRIPTION_PROPERTY); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/EnableReplicatedSubscriptionsIsDisabledTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/EnableReplicatedSubscriptionsIsDisabledTest.java index d002261cee4a3..98945a3594c60 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/EnableReplicatedSubscriptionsIsDisabledTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/EnableReplicatedSubscriptionsIsDisabledTest.java @@ -19,7 +19,7 @@ package org.apache.pulsar.client.api; import static org.assertj.core.api.Assertions.assertThat; -import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import java.util.Optional; @@ -28,7 +28,6 @@ import lombok.Cleanup; import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.Topic; -import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.common.naming.TopicName; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -74,9 +73,7 @@ public void testReplicateSubscriptionStateIsEnabled() throws Exception { Topic topicRef = optionalTopic.get(); Subscription subscription = topicRef.getSubscription(subName); assertNotNull(subscription); - assertTrue(subscription instanceof PersistentSubscription); - PersistentSubscription persistentSubscription = (PersistentSubscription) subscription; - assertEquals(persistentSubscription.getReplicatedControlled(), Boolean.FALSE); + assertFalse(subscription.isReplicated()); return true; }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ReplicateSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ReplicateSubscriptionTest.java index 327081bf1b9c8..1abad63573a3b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ReplicateSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ReplicateSubscriptionTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.client.api; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; @@ -35,6 +36,7 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +@Test(groups = "broker-api") public class ReplicateSubscriptionTest extends ProducerConsumerBase { @BeforeClass @@ -53,6 +55,9 @@ protected void cleanup() throws Exception { @Override protected void doInitConf() throws Exception { super.doInitConf(); + conf.setTopicLevelPoliciesEnabled(true); + conf.setSystemTopicEnabled(true); + conf.setEnableReplicatedSubscriptions(true); } @DataProvider @@ -87,10 +92,173 @@ public void testReplicateSubscriptionState(Boolean replicateSubscriptionState) Topic topicRef = optionalTopic.get(); Subscription subscription = topicRef.getSubscription(subName); assertNotNull(subscription); - assertTrue(subscription instanceof PersistentSubscription); - PersistentSubscription persistentSubscription = (PersistentSubscription) subscription; - assertEquals(persistentSubscription.getReplicatedControlled(), replicateSubscriptionState); + assertEquals(subscription.isReplicated(), replicateSubscriptionState != null + && replicateSubscriptionState); return true; }); } + + @DataProvider + public Object[][] replicateSubscriptionStateMultipleLevel() { + return new Object[][]{ + // consumer level high priority. + {Boolean.TRUE, Boolean.TRUE, Boolean.TRUE, true}, + {Boolean.TRUE, Boolean.TRUE, Boolean.TRUE, false}, + {Boolean.TRUE, Boolean.FALSE, Boolean.FALSE, true}, + {Boolean.TRUE, Boolean.FALSE, Boolean.FALSE, false}, + {Boolean.FALSE, Boolean.TRUE, Boolean.TRUE, true}, + {Boolean.FALSE, Boolean.TRUE, Boolean.TRUE, false}, + + // namespace level high priority + {null, Boolean.TRUE, null, true}, + {null, Boolean.TRUE, null, false}, + {null, Boolean.FALSE, null, true}, + {null, Boolean.FALSE, null, false}, + + // topic level high priority. + {null, Boolean.TRUE, Boolean.TRUE, true}, + {null, Boolean.TRUE, Boolean.TRUE, false}, + {null, Boolean.TRUE, Boolean.FALSE, true}, + {null, Boolean.TRUE, Boolean.FALSE, false}, + {null, Boolean.FALSE, Boolean.TRUE, true}, + {null, Boolean.FALSE, Boolean.TRUE, false}, + + // All higher levels are null. + {null, null, null, true}, + {null, null, null, false} + }; + } + + /** + * The priority order is as follows (from high to low): + * 1. Consumer/Subscription level + * 2. Topic level + * 3. Namespace level + * + * If the Consumer/Subscription level is set to false, it should be excluded from the evaluation process. + */ + @Test(dataProvider = "replicateSubscriptionStateMultipleLevel") + public void testReplicateSubscriptionStatePriority( + Boolean consumerReplicateSubscriptionState, + Boolean replicateSubscriptionEnabledOnNamespaceLevel, + Boolean replicateSubscriptionEnabledOnTopicLevel, + boolean replicatedSubscriptionStatus + ) throws Exception { + String nsName = "my-property/my-ns-" + System.nanoTime(); + admin.namespaces().createNamespace(nsName); + String topic = "persistent://" + nsName + "/" + System.nanoTime(); + String subName = "sub"; + @Cleanup + Consumer ignored = null; + ConsumerBuilder consumerBuilder = pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName(subName); + if (consumerReplicateSubscriptionState != null) { + consumerBuilder.replicateSubscriptionState(consumerReplicateSubscriptionState); + } + ignored = consumerBuilder.subscribe(); + + CompletableFuture> topicIfExists = pulsar.getBrokerService().getTopicIfExists(topic); + Optional topicOptional = topicIfExists.get(); + assertTrue(topicOptional.isPresent()); + Topic topicRef = topicOptional.get(); + Subscription subscription = topicRef.getSubscription(subName); + assertNotNull(subscription); + + assertEquals(subscription.isReplicated(), + consumerReplicateSubscriptionState != null && consumerReplicateSubscriptionState); + + // Verify the namespace level. + admin.namespaces().setReplicateSubscriptionState(nsName, replicateSubscriptionEnabledOnNamespaceLevel); + await().untilAsserted(() -> { + assertEquals(admin.namespaces().getReplicateSubscriptionState(nsName), + replicateSubscriptionEnabledOnNamespaceLevel); + assertEquals(admin.topicPolicies().getReplicateSubscriptionState(topic, true), + replicateSubscriptionEnabledOnNamespaceLevel); + if (Boolean.TRUE.equals(replicateSubscriptionEnabledOnNamespaceLevel)) { + assertTrue(subscription.isReplicated()); + } else { + // Using subscription policy. + assertEquals(subscription.isReplicated(), + consumerReplicateSubscriptionState != null && consumerReplicateSubscriptionState); + } + }); + + // Verify the topic level. + admin.topicPolicies().setReplicateSubscriptionState(topic, replicateSubscriptionEnabledOnTopicLevel); + await().untilAsserted(() -> { + assertEquals(admin.topicPolicies().getReplicateSubscriptionState(topic, false), + replicateSubscriptionEnabledOnTopicLevel); + Boolean replicateSubscriptionState = admin.topicPolicies().getReplicateSubscriptionState(topic, true); + assertTrue(replicateSubscriptionState == replicateSubscriptionEnabledOnTopicLevel + || replicateSubscriptionState == replicateSubscriptionEnabledOnNamespaceLevel); + if (consumerReplicateSubscriptionState == null || !consumerReplicateSubscriptionState) { + if (replicateSubscriptionEnabledOnTopicLevel != null) { + // Using topic policy. + assertEquals(subscription.isReplicated(), + replicateSubscriptionEnabledOnTopicLevel.booleanValue()); + } else { + // Using namespace policy. + assertEquals(subscription.isReplicated(), + replicateSubscriptionEnabledOnNamespaceLevel != null + && replicateSubscriptionEnabledOnNamespaceLevel); + } + } else { + // Using subscription policy. + assertTrue(subscription.isReplicated()); + } + }); + + // Verify the subscription level takes priority over the topic and namespace level. + admin.topics().setReplicatedSubscriptionStatus(topic, subName, replicatedSubscriptionStatus); + Boolean finalReplicateSubscriptionState; + + if (Boolean.TRUE.equals(replicatedSubscriptionStatus)) { + finalReplicateSubscriptionState = true; + } else if (replicateSubscriptionEnabledOnTopicLevel != null) { + finalReplicateSubscriptionState = replicateSubscriptionEnabledOnTopicLevel; + } else { + finalReplicateSubscriptionState = replicateSubscriptionEnabledOnNamespaceLevel; + } + + await().untilAsserted(() -> { + assertEquals(subscription.isReplicated(), + finalReplicateSubscriptionState != null && finalReplicateSubscriptionState); + assertTrue(subscription instanceof PersistentSubscription); + PersistentSubscription persistentSubscription = (PersistentSubscription) subscription; + boolean cursorFromReplicatedSubscription = + PersistentSubscription.isCursorFromReplicatedSubscription(persistentSubscription.getCursor()); + assertEquals(cursorFromReplicatedSubscription, replicatedSubscriptionStatus); + }); + } + + @Test(dataProvider = "replicateSubscriptionState") + public void testReplicateSubscriptionStateAfterUnload(Boolean replicateSubscriptionState) throws Exception { + String topic = "persistent://my-property/my-ns/" + System.nanoTime(); + String subName = "sub-" + System.nanoTime(); + ConsumerBuilder consumerBuilder = pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName(subName); + if (replicateSubscriptionState != null) { + consumerBuilder.replicateSubscriptionState(replicateSubscriptionState); + } + @Cleanup + Consumer ignored = consumerBuilder.subscribe(); + + admin.topics().unload(topic); + await().untilAsserted(() -> { + CompletableFuture> topicIfExists = pulsar.getBrokerService().getTopicIfExists(topic); + assertThat(topicIfExists) + .succeedsWithin(1, TimeUnit.SECONDS) + .matches(optionalTopic -> { + assertTrue(optionalTopic.isPresent()); + Topic topicRef = optionalTopic.get(); + Subscription subscription = topicRef.getSubscription(subName); + assertNotNull(subscription); + assertEquals(subscription.isReplicated(), replicateSubscriptionState != null + && replicateSubscriptionState); + return true; + }); + }); + } } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java index 28ad852064b4f..361ee60e180a8 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java @@ -4791,4 +4791,53 @@ void setIsAllowAutoUpdateSchema(String namespace, boolean isAllowAutoUpdateSchem */ CompletableFuture setNamespaceAllowedClustersAsync(String namespace, Set clusterIds); + + /** + * Enable or disable subscription replication on a namespace. + * + * @param namespace Namespace name + * @param enabled The replication status to set: + *
    + *
  • true: Enable subscription replication.
    • + *
  • false: Disable subscription replication.
    • + *
  • null: Remove config.
    • + *
+ */ + void setReplicateSubscriptionState(String namespace, Boolean enabled) throws PulsarAdminException; + + /** + * Enable or disable subscription replication on a namespace asynchronously. + * + * @param namespace Namespace name + * @param enabled The replication status to set: + *
    + *
  • true: Enable subscription replication.
    • + *
  • false: Disable subscription replication.
    • + *
  • null: Remove config.
    • + *
+ */ + CompletableFuture setReplicateSubscriptionStateAsync(String namespace, Boolean enabled); + + /** + * Get the enabled status of subscription replication on a namespace. + * + * @param namespace Namespace name + * @return true Subscription replication is enabled. + * false Subscription replication is disabled. + * null Subscription replication is not configured. + */ + Boolean getReplicateSubscriptionState(String namespace) throws PulsarAdminException; + + /** + * Get the enabled status of subscription replication on a namespace asynchronously. + * + * @param namespace Namespace name + * @return A {@link CompletableFuture} that will complete with the replication status: + *
    + *
  • true: Subscription replication is enabled.
    • + *
  • false: Subscription replication is disabled.
    • + *
  • null: Subscription replication is not configured.
    • + *
+ */ + CompletableFuture getReplicateSubscriptionStateAsync(String namespace); } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/TopicPolicies.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/TopicPolicies.java index 3e985dd728178..721012db061ca 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/TopicPolicies.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/TopicPolicies.java @@ -1950,4 +1950,109 @@ AutoSubscriptionCreationOverride getAutoSubscriptionCreation(String topic, * Delete topic policies, it works even if the topic has been deleted. */ void deleteTopicPolicies(String topic) throws PulsarAdminException; + + /** + * Get the ResourceGroup for a topic. + * + * @param topic Topic name + * @param applied True gets namespace level configuration if ResourceGroup does not exist on the topic. + * False gets topic level configuration. + * @return ResourceGroup + * @throws NotAuthorizedException Don't have admin permission + * @throws NotFoundException Topic does not exist + * @throws PulsarAdminException Unexpected error + */ + String getResourceGroup(String topic, boolean applied) throws PulsarAdminException; + + /** + * Get the ResourceGroup for a topic asynchronously. + * + * @param topic Topic name + */ + CompletableFuture getResourceGroupAsync(String topic, boolean applied); + + /** + * Set the ResourceGroup for a topic. + * + * @param topic Topic name + * @param resourceGroupName ResourceGroup name + * @throws NotAuthorizedException Don't have admin permission + * @throws NotFoundException Topic does not exist + * @throws PulsarAdminException Unexpected error + */ + void setResourceGroup(String topic, String resourceGroupName) throws PulsarAdminException; + + /** + * Set the ResourceGroup for a topic. + * + * @param topic Topic name + * @param resourceGroupName ResourceGroup name + */ + CompletableFuture setResourceGroupAsync(String topic, String resourceGroupName); + + /** + * Remove the ResourceGroup on a topic. + * + * @param topic Topic name + * @throws NotAuthorizedException Don't have admin permission + * @throws NotFoundException Topic does not exist + * @throws PulsarAdminException Unexpected error + */ + void removeResourceGroup(String topic) throws PulsarAdminException; + + /** + * Remove the ResourceGroup on a topic asynchronously. + * + * @param topic Topic name + */ + CompletableFuture removeResourceGroupAsync(String topic); + + /** + * Enable or disable subscription replication on a topic. + * + * @param topic Topic name + * @param enabled The replication status to set: + *
    + *
  • true: Enable subscription replication.
    • + *
  • false: Disable subscription replication.
    • + *
  • null: Remove config.
    • + *
+ */ + void setReplicateSubscriptionState(String topic, Boolean enabled) throws PulsarAdminException; + + /** + * Enable or disable subscription replication on a topic asynchronously. + * + * @param topic Topic name + * @param enabled The replication status to set: + *
    + *
  • true: Enable subscription replication.
    • + *
  • false: Disable subscription replication.
    • + *
  • null: Remove config.
    • + *
+ */ + CompletableFuture setReplicateSubscriptionStateAsync(String topic, Boolean enabled); + + /** + * Get the enabled status of subscription replication on a topic. + * + * @param topic Topic name + * @return true Subscription replication is enabled. + * false Subscription replication is disabled. + * null Subscription replication is not configured. + */ + Boolean getReplicateSubscriptionState(String topic, boolean applied) throws PulsarAdminException; + + /** + * Get the enabled status of subscription replication on a topic. + * + * @param topic Topic name + * @return A {@link CompletableFuture} that will complete with the replication status: + *
    + *
  • true: Subscription replication is enabled.
    • + *
  • false: Subscription replication is disabled.
    • + *
  • null: Subscription replication is not configured.
    • + *
+ */ + CompletableFuture getReplicateSubscriptionStateAsync(String topic, boolean applied); } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/Policies.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/Policies.java index a24df3e7ad442..51a9056d75609 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/Policies.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/Policies.java @@ -136,6 +136,9 @@ public class Policies { public Boolean dispatcherPauseOnAckStatePersistentEnabled; + @SuppressWarnings("checkstyle:MemberName") + public Boolean replicate_subscription_state; + public enum BundleType { LARGEST, HOT; } @@ -167,7 +170,8 @@ public int hashCode() { subscription_types_enabled, properties, resource_group_name, entryFilters, migrated, - dispatcherPauseOnAckStatePersistentEnabled); + dispatcherPauseOnAckStatePersistentEnabled, + replicate_subscription_state); } @Override @@ -218,7 +222,8 @@ public boolean equals(Object obj) { && Objects.equals(resource_group_name, other.resource_group_name) && Objects.equals(entryFilters, other.entryFilters) && Objects.equals(dispatcherPauseOnAckStatePersistentEnabled, - other.dispatcherPauseOnAckStatePersistentEnabled); + other.dispatcherPauseOnAckStatePersistentEnabled) + && Objects.equals(replicate_subscription_state, other.replicate_subscription_state); } return false; } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java index 7695abdd4809b..233d6a6328cc3 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java @@ -2042,5 +2042,27 @@ public CompletableFuture setNamespaceAllowedClustersAsync(String namespace return asyncPostRequest(path, Entity.entity(clusterIds, MediaType.APPLICATION_JSON)); } + @Override + public void setReplicateSubscriptionState(String namespace, Boolean enabled) throws PulsarAdminException { + sync(() -> setReplicateSubscriptionStateAsync(namespace, enabled)); + } + + @Override + public CompletableFuture setReplicateSubscriptionStateAsync(String namespace, Boolean enabled) { + NamespaceName ns = NamespaceName.get(namespace); + WebTarget path = namespacePath(ns, "replicateSubscriptionState"); + return asyncPostRequest(path, Entity.entity(enabled, MediaType.APPLICATION_JSON)); + } + @Override + public Boolean getReplicateSubscriptionState(String namespace) throws PulsarAdminException { + return sync(() -> getReplicateSubscriptionStateAsync(namespace)); + } + + @Override + public CompletableFuture getReplicateSubscriptionStateAsync(String namespace) { + NamespaceName ns = NamespaceName.get(namespace); + WebTarget path = namespacePath(ns, "replicateSubscriptionState"); + return asyncGetRequest(path, Boolean.class); + } } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicPoliciesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicPoliciesImpl.java index 6cfa981f1c46a..30f515228b5c7 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicPoliciesImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicPoliciesImpl.java @@ -1323,6 +1323,67 @@ public CompletableFuture deleteTopicPoliciesAsync(String topic) { return asyncDeleteRequest(path); } + @Override + public String getResourceGroup(String topic, boolean applied) throws PulsarAdminException { + return sync(() -> getResourceGroupAsync(topic, applied)); + } + + @Override + public CompletableFuture getResourceGroupAsync(String topic, boolean applied) { + TopicName tn = validateTopic(topic); + WebTarget path = topicPath(tn, "resourceGroup"); + path = path.queryParam("applied", applied); + return asyncGetRequest(path, String.class); + } + + @Override + public void setResourceGroup(String topic, String resourceGroupName) throws PulsarAdminException { + sync(() -> setResourceGroupAsync(topic, resourceGroupName)); + } + + @Override + public CompletableFuture setResourceGroupAsync(String topic, String resourceGroupName) { + TopicName tn = validateTopic(topic); + WebTarget path = topicPath(tn, "resourceGroup"); + return asyncPostRequest(path, Entity.entity(resourceGroupName, MediaType.APPLICATION_JSON_TYPE)); + } + + @Override + public void removeResourceGroup(String topic) throws PulsarAdminException { + sync(() -> removeResourceGroupAsync(topic)); + } + + @Override + public CompletableFuture removeResourceGroupAsync(String topic) { + return setResourceGroupAsync(topic, null); + } + + @Override + public void setReplicateSubscriptionState(String topic, Boolean enabled) throws PulsarAdminException { + sync(() -> setReplicateSubscriptionStateAsync(topic, enabled)); + } + + @Override + public CompletableFuture setReplicateSubscriptionStateAsync(String topic, Boolean enabled) { + TopicName topicName = validateTopic(topic); + WebTarget path = topicPath(topicName, "replicateSubscriptionState"); + return asyncPostRequest(path, Entity.entity(enabled, MediaType.APPLICATION_JSON)); + } + + @Override + public Boolean getReplicateSubscriptionState(String topic, boolean applied) + throws PulsarAdminException { + return sync(() -> getReplicateSubscriptionStateAsync(topic, applied)); + } + + @Override + public CompletableFuture getReplicateSubscriptionStateAsync(String topic, boolean applied) { + TopicName topicName = validateTopic(topic); + WebTarget path = topicPath(topicName, "replicateSubscriptionState"); + path = path.queryParam("applied", applied); + return asyncGetRequest(path, Boolean.class); + } + /* * returns topic name with encoded Local Name */ diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 8adedcd14ac40..54f6aa7214f0d 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -2665,6 +2665,46 @@ void run() throws PulsarAdminException { } } + @Command(description = "Set replicate subscription state from a namespace") + private class SetReplicateSubscriptionState extends CliCommand { + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; + + @Option(names = "--enabled", arity = "1", required = true, description = "Whether to replicate subscription" + + " state") + private boolean enabled = true; + + @Override + void run() throws PulsarAdminException { + String namespace = validateNamespace(namespaceName); + getAdmin().namespaces().setReplicateSubscriptionState(namespace, enabled); + } + } + + @Command(description = "Get replicate subscription state from a namespace") + private class GetReplicateSubscriptionState extends CliCommand { + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; + + @Override + void run() throws PulsarAdminException { + String namespace = validateNamespace(namespaceName); + print(getAdmin().namespaces().getReplicateSubscriptionState(namespace)); + } + } + + @Command(description = "Remove replicate subscription state from a namespace") + private class RemoveReplicateSubscriptionState extends CliCommand { + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; + + @Override + void run() throws PulsarAdminException { + String namespace = validateNamespace(namespaceName); + getAdmin().namespaces().setReplicateSubscriptionState(namespace, null); + } + } + public CmdNamespaces(Supplier admin) { super("namespaces", admin); addCommand("list", new GetNamespacesPerProperty()); @@ -2861,5 +2901,10 @@ public CmdNamespaces(Supplier admin) { new GetDispatcherPauseOnAckStatePersistent()); addCommand("remove-dispatcher-pause-on-ack-state-persistent", new RemoveDispatcherPauseOnAckStatePersistent()); + + + addCommand("set-replicate-subscription-state", new SetReplicateSubscriptionState()); + addCommand("get-replicate-subscription-state", new GetReplicateSubscriptionState()); + addCommand("remove-replicate-subscription-state", new RemoveReplicateSubscriptionState()); } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopicPolicies.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopicPolicies.java index 86a48cb20a77a..6d72b929cbf39 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopicPolicies.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopicPolicies.java @@ -173,6 +173,14 @@ public CmdTopicPolicies(Supplier admin) { addCommand("get-replication-clusters", new GetReplicationClusters()); addCommand("set-replication-clusters", new SetReplicationClusters()); addCommand("remove-replication-clusters", new RemoveReplicationClusters()); + + addCommand("set-resource-group", new SetResourceGroup()); + addCommand("get-resource-group", new GetResourceGroup()); + addCommand("remove-resource-group", new RemoveResourceGroup()); + + addCommand("set-replicate-subscription-state", new SetReplicateSubscriptionState()); + addCommand("get-replicate-subscription-state", new GetReplicateSubscriptionState()); + addCommand("remove-replicate-subscription-state", new RemoveReplicateSubscriptionState()); } @Command(description = "Get entry filters for a topic") @@ -2073,6 +2081,116 @@ void run() throws PulsarAdminException { } } + @Command(description = "Get ResourceGroup for a topic") + private class GetResourceGroup extends CliCommand { + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; + + @Option(names = {"--applied", "-a"}, description = "Get the applied policy of the topic") + private boolean applied = false; + + @Option(names = {"--global", "-g"}, description = "Whether to get this policy globally. " + + "If set to true, broker returned global topic policies") + private boolean isGlobal = false; + + @Override + void run() throws PulsarAdminException { + String persistentTopic = validatePersistentTopic(topicName); + print(getTopicPolicies(isGlobal).getResourceGroup(persistentTopic, applied)); + } + } + + @Command(description = "Set ResourceGroup for a topic") + private class SetResourceGroup extends CliCommand { + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; + + @Option(names = {"--global", "-g"}, description = "Whether to get this policy globally. " + + "If set to true, broker returned global topic policies") + private boolean isGlobal = false; + + @Option(names = {"--resource-group-name", "-rgn"}, description = "ResourceGroup name", required = true) + private String rgName; + + @Override + void run() throws PulsarAdminException { + String persistentTopic = validatePersistentTopic(topicName); + getTopicPolicies(isGlobal).setResourceGroup(persistentTopic, rgName); + } + } + + @Command(description = "Remove ResourceGroup from a topic") + private class RemoveResourceGroup extends CliCommand { + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; + + @Option(names = {"--global", "-g"}, description = "Whether to get this policy globally. " + + "If set to true, broker returned global topic policies") + private boolean isGlobal = false; + + + @Override + void run() throws PulsarAdminException { + String persistentTopic = validatePersistentTopic(topicName); + getTopicPolicies(isGlobal).removeResourceGroup(persistentTopic); + } + } + + @Command(description = "Set replicate subscription state from a topic") + private class SetReplicateSubscriptionState extends CliCommand { + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; + + @Option(names = {"--global", "-g"}, description = "Whether to get this policy globally. " + + "If set to true, broker returned global topic policies") + private boolean isGlobal = false; + + @Option(names = "--enabled", arity = "1", required = true, description = "Whether to replicate subscription" + + " state") + private boolean enabled = true; + + @Override + void run() throws PulsarAdminException { + String topic = validateTopicName(topicName); + getTopicPolicies(isGlobal).setReplicateSubscriptionState(topic, enabled); + } + } + + @Command(description = "Get replicate subscription state from a topic") + private class GetReplicateSubscriptionState extends CliCommand { + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; + + @Option(names = {"--global", "-g"}, description = "Whether to get this policy globally. " + + "If set to true, broker returned global topic policies") + private boolean isGlobal = false; + + @Option(names = {"--applied", "-a"}, description = "Get the applied policy of the topic") + private boolean applied = false; + + @Override + void run() throws PulsarAdminException { + String topic = validateTopicName(topicName); + print(getTopicPolicies(isGlobal).getReplicateSubscriptionState(topic, applied)); + } + } + + @Command(description = "Remove replicate subscription state from a topic") + private class RemoveReplicateSubscriptionState extends CliCommand { + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; + + @Option(names = {"--global", "-g"}, description = "Whether to get this policy globally. " + + "If set to true, broker returned global topic policies") + private boolean isGlobal = false; + + @Override + void run() throws PulsarAdminException { + String topic = validateTopicName(topicName); + getTopicPolicies(isGlobal).setReplicateSubscriptionState(topic, null); + } + } + private TopicPolicies getTopicPolicies(boolean isGlobal) { return getAdmin().topicPolicies(isGlobal); } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java index 4edb033498bc0..920eddcffbf72 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java @@ -62,6 +62,8 @@ public class HierarchyTopicPolicies { final PolicyHierarchyValue schemaValidationEnforced; final PolicyHierarchyValue entryFilters; + final PolicyHierarchyValue replicateSubscriptionState; + public HierarchyTopicPolicies() { replicationClusters = new PolicyHierarchyValue<>(); retentionPolicies = new PolicyHierarchyValue<>(); @@ -94,5 +96,6 @@ public HierarchyTopicPolicies() { dispatchRate = new PolicyHierarchyValue<>(); schemaValidationEnforced = new PolicyHierarchyValue<>(); entryFilters = new PolicyHierarchyValue<>(); + replicateSubscriptionState = new PolicyHierarchyValue<>(); } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/PolicyName.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/PolicyName.java index d77f92eb03292..be1e96b91c5d6 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/PolicyName.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/PolicyName.java @@ -58,4 +58,6 @@ public enum PolicyName { // cluster policies CLUSTER_MIGRATION, NAMESPACE_ISOLATION, + + REPLICATED_SUBSCRIPTION, } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java index 842c67714d5c7..46543b48c0ec5 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java @@ -89,6 +89,8 @@ public class TopicPolicies implements Cloneable { private Boolean schemaValidationEnforced; + private Boolean replicateSubscriptionState; + @SneakyThrows @Override public TopicPolicies clone() { diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/ReplicateSubscriptionStateTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/ReplicateSubscriptionStateTest.java new file mode 100644 index 0000000000000..2e2dcaf3c0f13 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/ReplicateSubscriptionStateTest.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.cli; + +import static org.awaitility.Awaitility.await; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertThrows; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.tests.integration.docker.ContainerExecException; +import org.apache.pulsar.tests.integration.docker.ContainerExecResult; +import org.apache.pulsar.tests.integration.suites.PulsarCliTestSuite; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +public class ReplicateSubscriptionStateTest extends PulsarCliTestSuite { + @BeforeClass(alwaysRun = true) + @Override + public void before() throws Exception { + enableTopicPolicies(); + super.before(); + } + + @AfterClass(alwaysRun = true) + @Override + public void after() throws Exception { + super.after(); + } + + @Test + public void testReplicateSubscriptionStateCmd() throws Exception { + TopicName topicName = TopicName.get(generateTopicName("testReplicateSubscriptionState", true)); + pulsarAdmin.topics().createNonPartitionedTopic(topicName.toString()); + String subName = "my-sub"; + pulsarAdmin.topics().createSubscription(topicName.toString(), subName, MessageId.earliest); + + String topicNameString = topicName.toString(); + String namesapceNameString = topicName.getNamespace(); + + ContainerExecResult result = pulsarCluster.runAdminCommandOnAnyBroker("namespaces", + "get-replicate-subscription-state", namesapceNameString); + assertEquals(result.getStdout().trim(), "null"); + assertEquals(result.getExitCode(), 0); + result = pulsarCluster.runAdminCommandOnAnyBroker("topicPolicies", + "get-schema-compatibility-strategy", topicNameString); + assertEquals(result.getStdout().trim(), "null"); + assertEquals(result.getExitCode(), 0); + + result = pulsarCluster.runAdminCommandOnAnyBroker("namespaces", + "set-replicate-subscription-state", "--enabled", "true", namesapceNameString); + assertEquals(result.getExitCode(), 0); + result = pulsarCluster.runAdminCommandOnAnyBroker("namespaces", + "get-replicate-subscription-state", namesapceNameString); + assertEquals(result.getExitCode(), 0); + assertEquals(result.getStdout().trim(), "true"); + result = pulsarCluster.runAdminCommandOnAnyBroker("namespaces", + "set-replicate-subscription-state", "--enabled", "false", namesapceNameString); + assertEquals(result.getExitCode(), 0); + result = pulsarCluster.runAdminCommandOnAnyBroker("namespaces", + "get-replicate-subscription-state", namesapceNameString); + assertEquals(result.getExitCode(), 0); + assertEquals(result.getStdout().trim(), "false"); + result = pulsarCluster.runAdminCommandOnAnyBroker("namespaces", + "remove-replicate-subscription-state", namesapceNameString); + assertEquals(result.getExitCode(), 0); + result = pulsarCluster.runAdminCommandOnAnyBroker("namespaces", + "get-replicate-subscription-state", namesapceNameString); + assertEquals(result.getExitCode(), 0); + assertEquals(result.getStdout().trim(), "null"); + + result = pulsarCluster.runAdminCommandOnAnyBroker("topicPolicies", "get-replicate-subscription-state", + "--applied", topicNameString); + assertEquals(result.getExitCode(), 0); + assertEquals(result.getStdout().trim(), "null"); + result = pulsarCluster.runAdminCommandOnAnyBroker("topicPolicies", "get-replicate-subscription-state", + topicNameString); + assertEquals(result.getExitCode(), 0); + assertEquals(result.getStdout().trim(), "null"); + + result = pulsarCluster.runAdminCommandOnAnyBroker("topicPolicies", + "set-replicate-subscription-state", "--enabled", "false", topicNameString); + assertEquals(result.getExitCode(), 0); + await().untilAsserted(() -> { + ContainerExecResult r = + pulsarCluster.runAdminCommandOnAnyBroker("topicPolicies", "get-replicate-subscription-state", + topicNameString); + assertEquals(r.getExitCode(), 0); + assertEquals(r.getStdout().trim(), "false"); + }); + await().untilAsserted(() -> { + ContainerExecResult r = + pulsarCluster.runAdminCommandOnAnyBroker("topicPolicies", "get-replicate-subscription-state", + "--applied", + topicNameString); + assertEquals(r.getExitCode(), 0); + assertEquals(r.getStdout().trim(), "false"); + }); + + result = pulsarCluster.runAdminCommandOnAnyBroker("topicPolicies", + "set-replicate-subscription-state", "--enabled", "true", topicNameString); + assertEquals(result.getExitCode(), 0); + await().untilAsserted(() -> { + ContainerExecResult r = + pulsarCluster.runAdminCommandOnAnyBroker("topicPolicies", "get-replicate-subscription-state", + topicNameString); + assertEquals(r.getExitCode(), 0); + assertEquals(r.getStdout().trim(), "true"); + }); + await().untilAsserted(() -> { + ContainerExecResult r = + pulsarCluster.runAdminCommandOnAnyBroker("topicPolicies", "get-replicate-subscription-state", + "--applied", + topicNameString); + assertEquals(r.getExitCode(), 0); + assertEquals(r.getStdout().trim(), "true"); + }); + + result = pulsarCluster.runAdminCommandOnAnyBroker("topicPolicies", + "remove-replicate-subscription-state", topicNameString); + assertEquals(result.getExitCode(), 0); + await().untilAsserted(() -> { + ContainerExecResult r = + pulsarCluster.runAdminCommandOnAnyBroker("topicPolicies", "get-replicate-subscription-state", + topicNameString); + assertEquals(r.getExitCode(), 0); + assertEquals(r.getStdout().trim(), "null"); + }); + } + + @Test + public void testReplicateSubscriptionStateCmdWithInvalidParameters() throws Exception { + assertThrows(ContainerExecException.class, () -> pulsarCluster.runAdminCommandOnAnyBroker("namespaces", + "set-replicate-subscription-state", "public/default")); + assertThrows(ContainerExecException.class, () -> pulsarCluster.runAdminCommandOnAnyBroker("namespaces", + "set-replicate-subscription-state", "--enabled", "public/default")); + + assertThrows(ContainerExecException.class, () -> pulsarCluster.runAdminCommandOnAnyBroker("topicPolicies", + "set-replicate-subscription-state", "public/default/test")); + assertThrows(ContainerExecException.class, () -> pulsarCluster.runAdminCommandOnAnyBroker("topicPolicies", + "set-replicate-subscription-state", "--enabled", "public/default/test")); + } +} \ No newline at end of file diff --git a/tests/integration/src/test/resources/pulsar-cli.xml b/tests/integration/src/test/resources/pulsar-cli.xml index af55aca8a0098..88e237790a2fb 100644 --- a/tests/integration/src/test/resources/pulsar-cli.xml +++ b/tests/integration/src/test/resources/pulsar-cli.xml @@ -34,6 +34,7 @@ + From 14038e14e5d683d25e806740a7b2c84547dade4c Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Wed, 25 Mar 2026 10:44:45 +0800 Subject: [PATCH 05/16] [fix][ci] Update Trivy action version to 0.35.0 --- .github/workflows/pulsar-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index d46a4923e9ddc..0907304bd0253 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -954,7 +954,7 @@ jobs: - name: Run Trivy container scan id: trivy_scan - uses: aquasecurity/trivy-action@0.26.0 + uses: aquasecurity/trivy-action@0.35.0 if: ${{ github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }} continue-on-error: true with: From 4d230ab31f8d328e50d3c9ddbe62fdf5c8a6fd6c Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Mon, 23 Mar 2026 18:37:53 +0800 Subject: [PATCH 06/16] [feat][broker] Add ResourceGroup-based rate limiting for replicator and topic --- .../pulsar/broker/ServiceConfiguration.java | 27 + .../resources/ResourceGroupResources.java | 9 + .../pulsar/broker/admin/AdminResource.java | 6 + .../broker/admin/impl/NamespacesBase.java | 16 +- .../admin/impl/PersistentTopicsBase.java | 98 +- .../broker/admin/impl/ResourceGroupsBase.java | 92 +- .../pulsar/broker/admin/v1/Namespaces.java | 8 +- .../pulsar/broker/admin/v2/Namespaces.java | 13 +- .../broker/admin/v2/PersistentTopics.java | 69 +- .../broker/admin/v2/ResourceGroups.java | 55 + .../broker/resourcegroup/ResourceGroup.java | 966 ++++++++++++------ .../ResourceGroupDispatchLimiter.java | 147 +++ .../ResourceGroupRateLimiterManager.java | 68 ++ .../resourcegroup/ResourceGroupService.java | 646 ++++++++---- .../ResourceQuotaCalculatorImpl.java | 90 +- .../service/AbstractBaseDispatcher.java | 71 +- .../pulsar/broker/service/AbstractTopic.java | 162 ++- .../pulsar/broker/service/Replicator.java | 5 + .../apache/pulsar/broker/service/Topic.java | 5 + .../nonpersistent/NonPersistentTopic.java | 4 +- .../persistent/DispatchRateLimiter.java | 14 +- .../persistent/GeoPersistentReplicator.java | 2 + .../persistent/PersistentReplicator.java | 80 +- .../service/persistent/PersistentTopic.java | 16 +- .../service/persistent/ShadowReplicator.java | 1 + .../src/main/proto/ResourceUsage.proto | 9 + .../AdminReplicatorDispatchRateTest.java | 210 ++++ .../broker/admin/AdminResourceGroupTest.java | 241 +++++ .../broker/admin/ResourceGroupsTest.java | 98 +- .../RGUsageMTAggrWaitForAllMsgsTest.java | 24 +- .../ResourceGroupMetricTest.java | 49 + .../ResourceGroupRateLimiterManagerTest.java | 161 +++ .../ResourceGroupRateLimiterTest.java | 5 + .../ResourceGroupReportLocalUsageTest.java | 112 +- .../ResourceGroupServiceTest.java | 570 ++++++++++- ...GroupUsageAggregationOnTopicLevelTest.java | 228 +++++ .../ResourceGroupUsageAggregationTest.java | 59 +- .../ResourceQuotaCalculatorImplTest.java | 13 +- .../ResourceUsageTransportManagerTest.java | 8 +- .../broker/service/AbstractTopicTest.java | 4 +- .../service/ReplicatorRateLimiterTest.java | 464 ++++++++- .../broker/transaction/TransactionTest.java | 1 + .../pulsar/client/admin/Namespaces.java | 71 ++ .../pulsar/client/admin/ResourceGroups.java | 54 + .../pulsar/client/admin/TopicPolicies.java | 62 ++ .../common/policies/data/ResourceGroup.java | 7 + .../client/admin/internal/NamespacesImpl.java | 40 +- .../admin/internal/ResourceGroupsImpl.java | 39 + .../admin/internal/TopicPoliciesImpl.java | 42 +- .../pulsar/admin/cli/PulsarAdminToolTest.java | 99 +- .../pulsar/admin/cli/CmdNamespaces.java | 18 +- .../pulsar/admin/cli/CmdResourceGroups.java | 83 ++ .../pulsar/admin/cli/CmdTopicPolicies.java | 18 +- .../policies/data/HierarchyTopicPolicies.java | 7 +- .../common/policies/data/TopicOperation.java | 3 + .../common/policies/data/TopicPolicies.java | 13 + .../src/main/resources/findbugsExclude.xml | 4 + 57 files changed, 4638 insertions(+), 848 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupDispatchLimiter.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManager.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminReplicatorDispatchRateTest.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminResourceGroupTest.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupMetricTest.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManagerTest.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationOnTopicLevelTest.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index dede4543fc30c..9314117bed5f5 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1268,6 +1268,33 @@ The max allowed delay for delayed delivery (in milliseconds). If the broker rece ) private int subscriptionPatternMaxLength = 50; + @FieldContext( + dynamic = true, + category = CATEGORY_POLICIES, + doc = "The percentage difference that is considered \"within limits\" to suppress usage reporting" + + "Setting this to 0 will also make us report in every round." + ) + private int resourceUsageReportSuppressionTolerancePercentage = 5; + + @FieldContext( + category = CATEGORY_POLICIES, + doc = "The maximum number of successive rounds that we can suppress reporting local usage, because there " + + "was no substantial change from the prior round. This is to ensure the reporting does not " + + "become too chatty. Set this value to one more than the cadence of sending reports; e.g., if " + + "you want to send every 3rd report, set the value to 4." + + "Setting this to 0 will make us report in every round." + + "Don't set to negative values; behavior will be disabled" + ) + private int resourceUsageMaxUsageReportSuppressRounds = 5; + + @FieldContext( + dynamic = true, + category = CATEGORY_POLICIES, + doc = "ResourceGroup rate limit will be triggered when the total traffic exceeds the product of the " + + "rate-limit value and the threshold. Value range: [0,1]." + ) + private double resourceGroupLocalQuotaThreshold = 0; + // <-- dispatcher read settings --> @FieldContext( dynamic = true, diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ResourceGroupResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ResourceGroupResources.java index 578609b21958f..13dec81ace6a6 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ResourceGroupResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ResourceGroupResources.java @@ -46,6 +46,10 @@ public boolean resourceGroupExists(String resourceGroupName) throws MetadataStor return exists(joinPath(BASE_PATH, resourceGroupName)); } + public CompletableFuture resourceGroupExistsAsync(String resourceGroupName) { + return existsAsync(joinPath(BASE_PATH, resourceGroupName)); + } + public void createResourceGroup(String resourceGroupName, ResourceGroup rg) throws MetadataStoreException { create(joinPath(BASE_PATH, resourceGroupName), rg); } @@ -60,6 +64,11 @@ public void updateResourceGroup(String resourceGroupName, set(joinPath(BASE_PATH, resourceGroupName), modifyFunction); } + public CompletableFuture updateResourceGroupAsync(String resourceGroupName, + Function modifyFunction) { + return setAsync(joinPath(BASE_PATH, resourceGroupName), modifyFunction); + } + public List listResourceGroups() throws MetadataStoreException { return getChildren(BASE_PATH); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java index bfa1fdc812b7b..6bdc33d9c5a15 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java @@ -49,6 +49,7 @@ import org.apache.pulsar.broker.service.TopicEventsListener.EventStage; import org.apache.pulsar.broker.service.TopicEventsListener.TopicEvent; import org.apache.pulsar.broker.service.TopicPoliciesService; +import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter; import org.apache.pulsar.broker.service.plugin.InvalidEntryFilterException; import org.apache.pulsar.broker.web.PulsarWebResource; import org.apache.pulsar.broker.web.RestException; @@ -1051,4 +1052,9 @@ protected CompletableFuture internalCheckTopicExists(TopicName topicName) } }); } + + protected String getReplicatorDispatchRateKey(String remoteCluster) { + return DispatchRateLimiter.getReplicatorDispatchRateKey(pulsar().getConfiguration().getClusterName(), + remoteCluster); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 024dc30e0ad1c..7476c381932f6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -2746,14 +2746,14 @@ protected CompletableFuture internalSetEntryFiltersPerTopicAsync(EntryFilt * Base method for setReplicatorDispatchRate v1 and v2. * Notion: don't re-use this logic. */ - protected void internalSetReplicatorDispatchRate(AsyncResponse asyncResponse, DispatchRateImpl dispatchRate) { + protected void internalSetReplicatorDispatchRate(AsyncResponse asyncResponse, String cluster, + DispatchRateImpl dispatchRate) { validateNamespacePolicyOperationAsync(namespaceName, PolicyName.REPLICATION_RATE, PolicyOperation.WRITE) .thenAccept(__ -> { log.info("[{}] Set namespace replicator dispatch-rate {}/{}", clientAppId(), namespaceName, dispatchRate); }).thenCompose(__ -> namespaceResources().setPoliciesAsync(namespaceName, policies -> { - String clusterName = pulsar().getConfiguration().getClusterName(); - policies.replicatorDispatchRate.put(clusterName, dispatchRate); + policies.replicatorDispatchRate.put(getReplicatorDispatchRateKey(cluster), dispatchRate); return policies; })).thenAccept(__ -> { asyncResponse.resume(Response.noContent().build()); @@ -2770,15 +2770,14 @@ protected void internalSetReplicatorDispatchRate(AsyncResponse asyncResponse, Di * Base method for getReplicatorDispatchRate v1 and v2. * Notion: don't re-use this logic. */ - protected void internalGetReplicatorDispatchRate(AsyncResponse asyncResponse) { + protected void internalGetReplicatorDispatchRate(AsyncResponse asyncResponse, String cluster) { validateNamespacePolicyOperationAsync(namespaceName, PolicyName.REPLICATION_RATE, PolicyOperation.READ) .thenCompose(__ -> namespaceResources().getPoliciesAsync(namespaceName)) .thenApply(policiesOpt -> { if (!policiesOpt.isPresent()) { throw new RestException(Response.Status.NOT_FOUND, "Namespace policies does not exist"); } - String clusterName = pulsar().getConfiguration().getClusterName(); - return policiesOpt.get().replicatorDispatchRate.get(clusterName); + return policiesOpt.get().replicatorDispatchRate.get(getReplicatorDispatchRateKey(cluster)); }).thenAccept(asyncResponse::resume) .exceptionally(ex -> { resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -2791,11 +2790,10 @@ protected void internalGetReplicatorDispatchRate(AsyncResponse asyncResponse) { * Base method for removeReplicatorDispatchRate v1 and v2. * Notion: don't re-use this logic. */ - protected void internalRemoveReplicatorDispatchRate(AsyncResponse asyncResponse) { + protected void internalRemoveReplicatorDispatchRate(AsyncResponse asyncResponse, String cluster) { validateNamespacePolicyOperationAsync(namespaceName, PolicyName.REPLICATION_RATE, PolicyOperation.WRITE) .thenCompose(__ -> namespaceResources().setPoliciesAsync(namespaceName, policies -> { - String clusterName = pulsar().getConfiguration().getClusterName(); - policies.replicatorDispatchRate.remove(clusterName); + policies.replicatorDispatchRate.remove(getReplicatorDispatchRateKey(cluster)); return policies; })).thenAccept(__ -> { asyncResponse.resume(Response.noContent().build()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 94bab3e41b1fc..5d4c5cb815e9f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -3673,24 +3673,68 @@ protected CompletableFuture internalSetMaxSubscriptionsPerTopic(Integer ma }); } - protected CompletableFuture internalGetReplicatorDispatchRate(boolean applied, boolean isGlobal) { + protected CompletableFuture internalGetReplicatorDispatchRate(String cluster, boolean applied, + boolean isGlobal) { return getTopicPoliciesAsyncWithRetry(topicName, isGlobal) - .thenApply(op -> op.map(TopicPolicies::getReplicatorDispatchRate) - .orElseGet(() -> { - if (applied) { - DispatchRateImpl namespacePolicy = getNamespacePolicies(namespaceName) - .replicatorDispatchRate.get(pulsar().getConfiguration().getClusterName()); - return namespacePolicy == null ? replicatorDispatchRate() : namespacePolicy; + .thenApply(op -> op.map(n -> { + // Prioritize getting the dispatch rate from the replicatorDispatchRateMap if a specific cluster + // is provided. + // If the cluster is empty, it means the user has not explicitly set a rate for a particular + // cluster, + // so we still attempt to retrieve the value from the replicatorDispatchRateMap using the current + // cluster. + // If `applied` is true, we also need to consider the default cluster rate and finally fallback + // to `getReplicatorDispatchRate()` for backward compatibility. + Map dispatchRateMap = new HashMap<>(); + if (n.getReplicatorDispatchRateMap() != null) { + dispatchRateMap = n.getReplicatorDispatchRateMap(); + } + DispatchRateImpl dispatchRate = dispatchRateMap.get(getReplicatorDispatchRateKey(cluster)); + if (dispatchRate != null) { + return dispatchRate; + } + + if (applied || StringUtils.isEmpty(cluster)) { + dispatchRate = + dispatchRateMap.get(pulsar().getConfiguration().getClusterName()); + if (dispatchRate != null) { + return dispatchRate; + } + // Backward compatibility. + return n.getReplicatorDispatchRate(); } return null; + }).orElseGet(() -> { + if (!applied) { + return null; + } + Map replicatorDispatchRate = + getNamespacePolicies(namespaceName).replicatorDispatchRate; + DispatchRateImpl namespacePolicy = + replicatorDispatchRate.getOrDefault(getReplicatorDispatchRateKey(cluster), + replicatorDispatchRate.get(pulsar().getConfiguration().getClusterName())); + return namespacePolicy == null ? replicatorDispatchRate() : namespacePolicy; })); } - protected CompletableFuture internalSetReplicatorDispatchRate(DispatchRateImpl dispatchRateToSet, + protected CompletableFuture internalSetReplicatorDispatchRate(String cluster, + DispatchRateImpl dispatchRateToSet, boolean isGlobal) { return pulsar().getTopicPoliciesService() .updateTopicPoliciesAsync(topicName, isGlobal, dispatchRateToSet == null, policies -> { - policies.setReplicatorDispatchRate(dispatchRateToSet); + if (policies.getReplicatorDispatchRateMap() == null) { + policies.setReplicatorDispatchRateMap(new HashMap<>()); + } + if (dispatchRateToSet == null) { + policies.getReplicatorDispatchRateMap() + .remove(getReplicatorDispatchRateKey(cluster)); + } else { + policies.getReplicatorDispatchRateMap() + .put(getReplicatorDispatchRateKey(cluster), dispatchRateToSet); + } + if (StringUtils.isEmpty(cluster)) { + policies.setReplicatorDispatchRate(dispatchRateToSet); + } }); } @@ -5504,4 +5548,40 @@ private static Long getIndexFromEntry(Entry entry) { return null; } } + + protected CompletableFuture internalSetResourceGroup(String resourceGroupName, boolean isGlobal) { + boolean isDelete = StringUtils.isEmpty(resourceGroupName); + return validateTopicOperationAsync(topicName, TopicOperation.SET_RESOURCE_GROUP) + .thenCompose(__ -> { + if (isDelete) { + return CompletableFuture.completedFuture(true); + } + return resourceGroupResources().resourceGroupExistsAsync(resourceGroupName); + }) + .thenCompose(exists -> { + if (!exists) { + return FutureUtil.failedFuture(new RestException(Status.NOT_FOUND, + "ResourceGroup does not exist")); + } + + return pulsar().getTopicPoliciesService() + .updateTopicPoliciesAsync(topicName, isGlobal, false, + policies -> { + policies.setResourceGroupName(isDelete ? null : resourceGroupName); + }); + }); + } + + protected CompletableFuture internalGetResourceGroup(boolean applied, boolean isGlobal) { + return validateTopicOperationAsync(topicName, TopicOperation.GET_RESOURCE_GROUP) + .thenCompose(__ -> getTopicPoliciesAsyncWithRetry(topicName, isGlobal) + .thenApply(op -> op.map(TopicPolicies::getResourceGroupName) + .orElseGet(() -> { + if (applied) { + return getNamespacePolicies(namespaceName).resource_group_name; + } + return null; + }) + )); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ResourceGroupsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ResourceGroupsBase.java index 826b9c322e353..f900d23aee4ac 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ResourceGroupsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ResourceGroupsBase.java @@ -19,17 +19,69 @@ package org.apache.pulsar.broker.admin.impl; import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; import org.apache.pulsar.broker.admin.AdminResource; import org.apache.pulsar.broker.web.RestException; -import org.apache.pulsar.common.naming.NamespaceName; -import org.apache.pulsar.common.policies.data.Policies; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.ResourceGroup; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public abstract class ResourceGroupsBase extends AdminResource { + + protected CompletableFuture internalSetReplicatorDispatchRate(String rgName, String remoteCluster, + DispatchRate dispatchRate) { + if (remoteCluster == null) { + return FutureUtil.failedFuture(new RestException(Status.PRECONDITION_FAILED, + "Remote cluster name is not provided")); + } + return validateSuperUserAccessAsync() + .thenCompose((__) -> resourceGroupResources() + .updateResourceGroupAsync(rgName, rg -> { + String key = getReplicatorDispatchRateKey(remoteCluster); + if (rg.getReplicatorDispatchRate() == null) { + rg.setReplicatorDispatchRate(new ConcurrentHashMap<>()); + } + if (dispatchRate == null) { + rg.getReplicatorDispatchRate().remove(key); + } else { + rg.getReplicatorDispatchRate().put(key, dispatchRate); + } + return rg; + }) + ); + } + + protected CompletableFuture internalGetReplicatorDispatchRate(String rgName, String remoteCluster) { + if (remoteCluster == null) { + return FutureUtil.failedFuture(new RestException(Status.PRECONDITION_FAILED, + "Remote cluster name is not provided")); + } + return validateSuperUserAccessAsync() + .thenCompose((__) -> resourceGroupResources() + .getResourceGroupAsync(rgName)) + .thenCompose((resourceGroupOptional) -> resourceGroupOptional + .map((rg) -> { + Map replicatorDispatchRate = rg.getReplicatorDispatchRate(); + DispatchRate dispatchRate = null; + if (replicatorDispatchRate != null) { + dispatchRate = rg.getReplicatorDispatchRate() + .get(getReplicatorDispatchRateKey(remoteCluster)); + } + return CompletableFuture.completedFuture(dispatchRate); + }) + .orElseGet(() -> FutureUtil.failedFuture( + new RestException(Status.NOT_FOUND, "ResourceGroup does not exist"))) + ); + } + protected List internalGetResourceGroups() { try { validateSuperUserAccess(); @@ -75,7 +127,12 @@ protected void internalUpdateResourceGroup(String rgName, ResourceGroup rgConfig if (rgConfig.getDispatchRateInBytes() != null) { resourceGroup.setDispatchRateInBytes(rgConfig.getDispatchRateInBytes()); } - + if (rgConfig.getReplicationDispatchRateInBytes() != null) { + resourceGroup.setReplicationDispatchRateInBytes(rgConfig.getReplicationDispatchRateInBytes()); + } + if (rgConfig.getReplicationDispatchRateInMsgs() != null) { + resourceGroup.setReplicationDispatchRateInMsgs(rgConfig.getReplicationDispatchRateInMsgs()); + } // write back the new ResourceGroup config. resourceGroupResources().updateResourceGroup(rgName, r -> resourceGroup); log.info("[{}] Successfully updated the ResourceGroup {}", clientAppId(), rgName); @@ -96,6 +153,10 @@ protected void internalCreateResourceGroup(String rgName, ResourceGroup rgConfig ? -1 : rgConfig.getDispatchRateInMsgs()); rgConfig.setDispatchRateInBytes(rgConfig.getDispatchRateInBytes() == null ? -1 : rgConfig.getDispatchRateInBytes()); + rgConfig.setReplicationDispatchRateInBytes(rgConfig.getReplicationDispatchRateInBytes() == null + ? -1 : rgConfig.getReplicationDispatchRateInBytes()); + rgConfig.setReplicationDispatchRateInMsgs(rgConfig.getReplicationDispatchRateInMsgs() == null + ? -1 : rgConfig.getReplicationDispatchRateInMsgs()); try { resourceGroupResources().createResourceGroup(rgName, rgConfig); log.info("[{}] Created ResourceGroup {}", clientAppId(), rgName); @@ -138,23 +199,6 @@ protected void internalCreateOrUpdateResourceGroup(String rgName, ResourceGroup } } - protected boolean internalCheckRgInUse(String rgName) { - try { - for (String tenant : tenantResources().listTenants()) { - for (String namespace : tenantResources().getListOfNamespaces(tenant)) { - Policies policies = getNamespacePolicies(NamespaceName.get(namespace)); - if (null != policies && rgName.equals(policies.resource_group_name)) { - return true; - } - } - } - } catch (Exception e) { - log.error("[{}] Failed to get tenant/namespace list {}: {}", clientAppId(), rgName, e); - throw new RestException(e); - } - return false; - } - protected void internalDeleteResourceGroup(String rgName) { /* * need to walk the namespaces and make sure it is not in use @@ -164,8 +208,12 @@ protected void internalDeleteResourceGroup(String rgName) { /* * walk the namespaces and make sure it is not in use. */ - if (internalCheckRgInUse(rgName)) { - throw new RestException(Response.Status.PRECONDITION_FAILED, "ResourceGroup is in use"); + try { + pulsar().getResourceGroupServiceManager().checkResourceGroupInUse(rgName); + } catch (PulsarAdminException e) { + log.error("[{}] Check if ResourceGroup {} is in use: {}", clientAppId(), rgName, e); + throw new RestException(Response.Status.PRECONDITION_FAILED, + "ResourceGroup is in use"); } resourceGroupResources().deleteResourceGroup(rgName); log.info("[{}] Deleted ResourceGroup {}", clientAppId(), rgName); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java index 276ea457d2cf4..9c79871571cb9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java @@ -1118,10 +1118,11 @@ public void setReplicatorDispatchRate(@Suspended AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, + @QueryParam("cluster") String queryCluster, @ApiParam(value = "Replicator dispatch rate for all topics of the specified namespace") DispatchRateImpl dispatchRate) { validateNamespaceName(tenant, cluster, namespace); - internalSetReplicatorDispatchRate(asyncResponse, dispatchRate); + internalSetReplicatorDispatchRate(asyncResponse, cluster, dispatchRate); } @GET @@ -1135,9 +1136,10 @@ public void getReplicatorDispatchRate( @Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("cluster") String cluster, - @PathParam("namespace") String namespace) { + @PathParam("namespace") String namespace, + @QueryParam("cluster") String queryCluster) { validateNamespaceName(tenant, cluster, namespace); - internalGetReplicatorDispatchRate(asyncResponse); + internalGetReplicatorDispatchRate(asyncResponse, cluster); } @GET diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index f172e36ff6d5a..7dad08416e117 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -1235,9 +1235,10 @@ public void getSubscribeRate(@Suspended AsyncResponse asyncResponse, @PathParam( @ApiResponse(code = 403, message = "Don't have admin permission")}) public void removeReplicatorDispatchRate(@Suspended AsyncResponse asyncResponse, @PathParam("tenant") String tenant, - @PathParam("namespace") String namespace) { + @PathParam("namespace") String namespace, + @QueryParam("cluster") String cluster) { validateNamespaceName(tenant, namespace); - internalRemoveReplicatorDispatchRate(asyncResponse); + internalRemoveReplicatorDispatchRate(asyncResponse, cluster); } @POST @@ -1249,10 +1250,11 @@ public void removeReplicatorDispatchRate(@Suspended AsyncResponse asyncResponse, public void setReplicatorDispatchRate(@Suspended AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, + @QueryParam("cluster") String cluster, @ApiParam(value = "Replicator dispatch rate for all topics of the specified namespace") DispatchRateImpl dispatchRate) { validateNamespaceName(tenant, namespace); - internalSetReplicatorDispatchRate(asyncResponse, dispatchRate); + internalSetReplicatorDispatchRate(asyncResponse, cluster, dispatchRate); } @GET @@ -1264,9 +1266,10 @@ public void setReplicatorDispatchRate(@Suspended AsyncResponse asyncResponse, @ApiResponse(code = 404, message = "Namespace does not exist") }) public void getReplicatorDispatchRate(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, - @PathParam("namespace") String namespace) { + @PathParam("namespace") String namespace, + @QueryParam("cluster") String cluster) { validateNamespaceName(tenant, namespace); - internalGetReplicatorDispatchRate(asyncResponse); + internalGetReplicatorDispatchRate(asyncResponse, cluster); } @GET diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java index 3b854e49f47cb..3706194d93199 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java @@ -3035,11 +3035,12 @@ public void getReplicatorDispatchRate(@Suspended final AsyncResponse asyncRespon @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal, @QueryParam("applied") @DefaultValue("false") boolean applied, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") - @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, + @QueryParam("cluster") String cluster) { validateTopicName(tenant, namespace, encodedTopic); validateTopicPolicyOperationAsync(topicName, PolicyName.REPLICATION_RATE, PolicyOperation.READ) .thenCompose(__ -> preValidation(authoritative)) - .thenCompose(__ -> internalGetReplicatorDispatchRate(applied, isGlobal)) + .thenCompose(__ -> internalGetReplicatorDispatchRate(cluster, applied, isGlobal)) .thenApply(asyncResponse::resume) .exceptionally(ex -> { handleTopicPolicyException("getReplicatorDispatchRate", ex, asyncResponse); @@ -3065,11 +3066,12 @@ public void setReplicatorDispatchRate(@Suspended final AsyncResponse asyncRespon @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, + @QueryParam("cluster") String cluster, @ApiParam(value = "Replicator dispatch rate of the topic") DispatchRateImpl dispatchRate) { validateTopicName(tenant, namespace, encodedTopic); validateTopicPolicyOperationAsync(topicName, PolicyName.REPLICATION_RATE, PolicyOperation.WRITE) .thenCompose(__ -> preValidation(authoritative)) - .thenCompose(__ -> internalSetReplicatorDispatchRate(dispatchRate, isGlobal)) + .thenCompose(__ -> internalSetReplicatorDispatchRate(cluster, dispatchRate, isGlobal)) .thenRun(() -> { log.info("[{}] Successfully updated replicatorDispatchRate: namespace={}, topic={}" + ", replicatorDispatchRate={}, isGlobal={}", @@ -3098,11 +3100,12 @@ public void removeReplicatorDispatchRate(@Suspended final AsyncResponse asyncRes @PathParam("topic") @Encoded String encodedTopic, @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") - @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, + @QueryParam("cluster") String cluster) { validateTopicName(tenant, namespace, encodedTopic); validateTopicPolicyOperationAsync(topicName, PolicyName.REPLICATION_RATE, PolicyOperation.WRITE) .thenCompose(__ -> preValidation(authoritative)) - .thenCompose(__ -> internalSetReplicatorDispatchRate(null, isGlobal)) + .thenCompose(__ -> internalSetReplicatorDispatchRate(cluster, null, isGlobal)) .thenRun(() -> { log.info("[{}] Successfully remove replicatorDispatchRate limit: namespace={}, topic={}", clientAppId(), namespaceName, topicName.getLocalName()); @@ -5207,5 +5210,61 @@ public void getMessageIDByIndex(@Suspended final AsyncResponse asyncResponse, }); } + @POST + @Path("/{tenant}/{namespace}/{topic}/resourceGroup") + @ApiOperation(value = "Set ResourceGroup for a topic") + @ApiResponses(value = { + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Topic doesn't exist"), + @ApiResponse(code = 405, message = + "Topic level policy is disabled, enable the topic level policy and retry"), + @ApiResponse(code = 409, message = "Concurrent modification") + }) + public void setResourceGroup( + @Suspended final AsyncResponse asyncResponse, + @PathParam("tenant") String tenant, + @PathParam("namespace") String namespace, + @PathParam("topic") String encodedTopic, + @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal, + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, + @ApiParam(value = "ResourceGroup name", required = true) String resourceGroupName) { + validateTopicName(tenant, namespace, encodedTopic); + preValidation(authoritative) + .thenCompose(__ -> internalSetResourceGroup(resourceGroupName, isGlobal)) + .thenAccept(__ -> asyncResponse.resume(Response.noContent().build())) + .exceptionally(ex -> { + handleTopicPolicyException("setResourceGroup", ex, asyncResponse); + return null; + }); + } + + @GET + @Path("/{tenant}/{namespace}/{topic}/resourceGroup") + @ApiOperation(value = "Get ResourceGroup for a topic") + @ApiResponses(value = { + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Topic doesn't exist"), + @ApiResponse(code = 405, message = + "Topic level policy is disabled, enable the topic level policy and retry"), + @ApiResponse(code = 409, message = "Concurrent modification") + }) + public void getResourceGroup( + @Suspended final AsyncResponse asyncResponse, + @PathParam("tenant") String tenant, + @PathParam("namespace") String namespace, + @PathParam("topic") String encodedTopic, + @QueryParam("applied") @DefaultValue("false") boolean applied, + @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal, + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + validateTopicName(tenant, namespace, encodedTopic); + preValidation(authoritative) + .thenCompose(__ -> internalGetResourceGroup(applied, isGlobal)) + .thenApply(asyncResponse::resume) + .exceptionally(ex -> { + handleTopicPolicyException("getResourceGroup", ex, asyncResponse); + return null; + }); + } + private static final Logger log = LoggerFactory.getLogger(PersistentTopics.class); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ResourceGroups.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ResourceGroups.java index 58f593e20ce3b..29e46f0ae5098 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ResourceGroups.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ResourceGroups.java @@ -17,6 +17,7 @@ * under the License. */ package org.apache.pulsar.broker.admin.v2; + import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; @@ -26,12 +27,18 @@ import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.GET; +import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.container.AsyncResponse; +import javax.ws.rs.container.Suspended; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import org.apache.pulsar.broker.admin.impl.ResourceGroupsBase; +import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.ResourceGroup; @Path("/resourcegroups") @@ -81,5 +88,53 @@ public void createOrUpdateResourceGroup(@PathParam("resourcegroup") String name, public void deleteResourceGroup(@PathParam("resourcegroup") String resourcegroup) { internalDeleteResourceGroup(resourcegroup); } + + @POST + @Path("/{resourcegroup}/replicatorDispatchRate") + @ApiOperation(value = "Set replicator dispatch-rate throttling for a resourcegroup") + @ApiResponses(value = { + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "ResourceGroup doesn't exist")}) + public void setReplicatorDispatchRate(@Suspended AsyncResponse asyncResponse, + @PathParam("resourcegroup") String resourcegroup, + @QueryParam("cluster") String remoteCluster, + DispatchRate dispatchRate) { + internalSetReplicatorDispatchRate(resourcegroup, remoteCluster, dispatchRate) + .thenAccept((__) -> asyncResponse.resume(Response.noContent().build())) + .exceptionally(ex -> { + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); + } + + @DELETE + @Path("/{resourcegroup}/replicatorDispatchRate") + @ApiOperation(value = "Delete replicator dispatch-rate throttling for a resourcegroup") + @ApiResponses(value = { + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "ResourceGroup doesn't exist")}) + public void removeReplicatorDispatchRate(@Suspended AsyncResponse asyncResponse, + @PathParam("resourcegroup") String resourcegroup, + @QueryParam("cluster") String remoteCluster, + DispatchRate dispatchRate) { + setReplicatorDispatchRate(asyncResponse, resourcegroup, remoteCluster, null); + } + + @GET + @Path("/{resourcegroup}/replicatorDispatchRate") + @ApiOperation(value = "Get replicator dispatch-rate throttling for a resourcegroup") + @ApiResponses(value = { + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "ResourceGroup doesn't exist")}) + public void getReplicatorDispatchRate(@Suspended AsyncResponse asyncResponse, + @PathParam("resourcegroup") String resourcegroup, + @QueryParam("cluster") String remoteCluster) { + internalGetReplicatorDispatchRate(resourcegroup, remoteCluster) + .thenAccept(asyncResponse::resume) + .exceptionally(ex -> { + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroup.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroup.java index 55cda0ca9dde3..dfa43888deae7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroup.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroup.java @@ -18,19 +18,36 @@ */ package org.apache.pulsar.broker.resourcegroup; +import static java.util.Objects.requireNonNull; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.annotations.VisibleForTesting; import io.prometheus.client.Counter; +import io.prometheus.client.Gauge; import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; +import lombok.AllArgsConstructor; import lombok.Getter; -import lombok.val; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.resourcegroup.ResourceGroupService.ResourceGroupOpStatus; +import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter; import org.apache.pulsar.broker.service.resource.usage.NetworkUsage; +import org.apache.pulsar.broker.service.resource.usage.ReplicatorUsage; import org.apache.pulsar.broker.service.resource.usage.ResourceUsage; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.common.policies.data.DispatchRate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,11 +61,33 @@ * same RG across all of the monitoring classes it uses (Publish/Dispatch/...), instead of referencing one RG for * publish, another one for dispatch, etc. */ -public class ResourceGroup { +public class ResourceGroup implements AutoCloseable{ + + private volatile boolean isClosed = false; + + @Override + public void close() throws Exception { + synchronized (this) { + isClosed = true; + if (resourceGroupDispatchLimiter != null) { + resourceGroupDispatchLimiter.close(); + } + if (resourceGroupReplicationDispatchLimiter != null) { + resourceGroupReplicationDispatchLimiter.close(); + } + replicatorDispatchRateLimiterMap.forEach((k, v) -> { + v.close(); + notifyReplicatorDispatchRateLimiterConsumer(k, null); + }); + } + } /** * Convenience class for bytes and messages counts, which are used together in a lot of the following code. */ + @ToString + @AllArgsConstructor + @NoArgsConstructor public static class BytesAndMessagesCount { public long bytes; public long messages; @@ -61,6 +100,7 @@ public static class BytesAndMessagesCount { public enum ResourceGroupMonitoringClass { Publish, Dispatch, + ReplicationDispatch, // Storage; // Punt this for now, until we have a clearer idea of the usage, statistics, etc. } @@ -70,22 +110,14 @@ public enum ResourceGroupMonitoringClass { public enum ResourceGroupRefTypes { Tenants, Namespaces, - // Topics; // Punt this for when we support direct ref/under from topics. + Topics } // Default ctor: it is not expected that anything outside of this package will need to directly // construct a ResourceGroup (i.e., without going through ResourceGroupService). protected ResourceGroup(ResourceGroupService rgs, String name, org.apache.pulsar.common.policies.data.ResourceGroup rgConfig) { - this.rgs = rgs; - this.resourceGroupName = name; - this.setResourceGroupMonitoringClassFields(); - this.setResourceGroupConfigParameters(rgConfig); - this.setDefaultResourceUsageTransportHandlers(); - this.resourceGroupPublishLimiter = new ResourceGroupPublishLimiter(rgConfig, rgs.getPulsar() - .getMonotonicClock()); - log.info("attaching publish rate limiter {} to {} get {}", this.resourceGroupPublishLimiter, name, - this.getResourceGroupPublishLimiter()); + this(rgs, name, rgConfig, null, null); } // ctor for overriding the transport-manager fill/set buffer. @@ -96,64 +128,137 @@ protected ResourceGroup(ResourceGroupService rgs, String rgName, ResourceUsagePublisher rgPublisher, ResourceUsageConsumer rgConsumer) { this.rgs = rgs; this.resourceGroupName = rgName; - this.setResourceGroupMonitoringClassFields(); - this.setResourceGroupConfigParameters(rgConfig); - this.resourceGroupPublishLimiter = new ResourceGroupPublishLimiter(rgConfig, rgs.getPulsar() - .getMonotonicClock()); - this.ruPublisher = rgPublisher; - this.ruConsumer = rgConsumer; + this.rgConfig = rgConfig; + if (rgPublisher == null && rgConsumer == null) { + this.setDefaultResourceUsageTransportHandlers(); + } else { + this.ruPublisher = rgPublisher; + this.ruConsumer = rgConsumer; + } + this.resourceGroupPublishLimiter = + new ResourceGroupPublishLimiter(rgConfig, rgs.getPulsar().getMonotonicClock()); + log.info("attaching publish rate limiter {} to {} get {}", this.resourceGroupPublishLimiter.toString(), rgName, + this.getResourceGroupPublishLimiter()); + this.resourceGroupReplicationDispatchLimiter = ResourceGroupRateLimiterManager + .newReplicationDispatchRateLimiter(rgConfig); + log.info("attaching replication dispatch rate limiter {} to {}", this.resourceGroupReplicationDispatchLimiter, + rgName); + this.resourceGroupDispatchLimiter = ResourceGroupRateLimiterManager + .newDispatchRateLimiter(rgConfig); + log.info("attaching topic dispatch rate limiter {} to {}", this.resourceGroupDispatchLimiter, rgName); } - // copy ctor: note does shallow copy. - // The envisioned usage is for display of an RG's state, without scope for direct update. - public ResourceGroup(ResourceGroup other) { - this.resourceGroupName = other.resourceGroupName; - this.rgs = other.rgs; - this.resourceGroupPublishLimiter = other.resourceGroupPublishLimiter; - this.setResourceGroupMonitoringClassFields(); - - // ToDo: copy the monitoring class fields, and ruPublisher/ruConsumer from other, if required. - - this.resourceGroupNamespaceRefs = other.resourceGroupNamespaceRefs; - this.resourceGroupTenantRefs = other.resourceGroupTenantRefs; - - for (int idx = 0; idx < ResourceGroupMonitoringClass.values().length; idx++) { - PerMonitoringClassFields thisFields = this.monitoringClassFields[idx]; - PerMonitoringClassFields otherFields = other.monitoringClassFields[idx]; - - thisFields.configValuesPerPeriod.bytes = otherFields.configValuesPerPeriod.bytes; - thisFields.configValuesPerPeriod.messages = otherFields.configValuesPerPeriod.messages; - - thisFields.quotaForNextPeriod.bytes = otherFields.quotaForNextPeriod.bytes; - thisFields.quotaForNextPeriod.messages = otherFields.quotaForNextPeriod.messages; + protected void updateResourceGroup(org.apache.pulsar.common.policies.data.ResourceGroup rgConfig) { + synchronized (this) { + if (isClosed) { + return; + } + updateMonitoringClassFieldsMap(rgConfig); + BytesAndMessagesCount pubBmc = new BytesAndMessagesCount(); + pubBmc.messages = rgConfig.getPublishRateInMsgs() == null ? -1 : rgConfig.getPublishRateInMsgs(); + pubBmc.bytes = rgConfig.getPublishRateInBytes() == null ? -1 : rgConfig.getPublishRateInBytes(); + this.resourceGroupPublishLimiter.update(pubBmc); + updateReplicationDispatchLimiters(rgConfig); + } + } - thisFields.usedLocallySinceLastReport.bytes = otherFields.usedLocallySinceLastReport.bytes; - thisFields.usedLocallySinceLastReport.messages = otherFields.usedLocallySinceLastReport.messages; + private void updateReplicationDispatchLimiters(org.apache.pulsar.common.policies.data.ResourceGroup rgConfig) { + ResourceGroupRateLimiterManager + .updateReplicationDispatchRateLimiter(resourceGroupReplicationDispatchLimiter, rgConfig); + replicatorDispatchRateLimiterMap.forEach((key, oldLimiter) -> + replicatorDispatchRateLimiterMap.computeIfPresent(key, + (k, curr) -> updateReplicatorLimiter(k, curr, rgConfig)) + ); + } + + private ResourceGroupDispatchLimiter updateReplicatorLimiter(String key, ResourceGroupDispatchLimiter currLimiter, + org.apache.pulsar.common.policies.data.ResourceGroup rgConfig) { + DispatchRate dispatchRate = rgConfig.getReplicatorDispatchRate().get(key); + if (dispatchRate == null) { + // Specific rate-limiter for this cluster is removed in the new config. + // use the default rate-limiter, notify the rate-limiter consumers. + if (currLimiter != resourceGroupReplicationDispatchLimiter) { + notifyReplicatorDispatchRateLimiterConsumer(key, resourceGroupReplicationDispatchLimiter); + return resourceGroupReplicationDispatchLimiter; + } + } else if (currLimiter == resourceGroupReplicationDispatchLimiter) { + // Specific rate-limiter for this cluster is provided in the new config. + // When rate-limiter is default, new a rate-limiter. + ResourceGroupDispatchLimiter newLimiter = createNewReplicatorLimiter(key); + notifyReplicatorDispatchRateLimiterConsumer(key, newLimiter); + return newLimiter; + } else { + currLimiter.update(dispatchRate.getDispatchThrottlingRateInMsg(), + dispatchRate.getDispatchThrottlingRateInByte()); + } + return currLimiter; + } - thisFields.lastResourceUsageFillTimeMSecsSinceEpoch = otherFields.lastResourceUsageFillTimeMSecsSinceEpoch; + protected String getReplicatorDispatchRateLimiterKey(String remoteCluster) { + return DispatchRateLimiter.getReplicatorDispatchRateKey(rgs.getPulsar().getConfiguration().getClusterName(), + remoteCluster); + } - thisFields.numSuppressedUsageReports = otherFields.numSuppressedUsageReports; + private ResourceGroupDispatchLimiter createNewReplicatorLimiter(String key) { + return Optional.ofNullable(rgConfig.getReplicatorDispatchRate().get(key)) + .map(dispatchRate -> ResourceGroupRateLimiterManager.newReplicationDispatchRateLimiter( + dispatchRate.getDispatchThrottlingRateInMsg(), + dispatchRate.getDispatchThrottlingRateInByte())) + .orElse(resourceGroupReplicationDispatchLimiter); + } - thisFields.totalUsedLocally.bytes = otherFields.totalUsedLocally.bytes; - thisFields.totalUsedLocally.messages = otherFields.totalUsedLocally.messages; + private void notifyReplicatorDispatchRateLimiterConsumer( + String key, ResourceGroupDispatchLimiter limiter) { + replicatorDispatchRateLimiterConsumerMap.computeIfPresent(key, (__, consumerSet) -> { + consumerSet.forEach(c -> c.accept(limiter)); + return consumerSet; + }); + } - // ToDo: Deep copy instead? - thisFields.usageFromOtherBrokers = otherFields.usageFromOtherBrokers; + public void registerReplicatorDispatchRateLimiter(String remoteCluster, + Consumer consumer) { + String key = getReplicatorDispatchRateLimiterKey(remoteCluster); + synchronized (this) { + if (isClosed) { + unregisterReplicatorDispatchRateLimiter(remoteCluster, consumer); + // The resource group is closed, no need to register the rate limiter. + return; + } + ResourceGroupDispatchLimiter limiter = + replicatorDispatchRateLimiterMap.computeIfAbsent(key, __ -> createNewReplicatorLimiter(key)); + consumer.accept(limiter); + // Must use compute instead of computeIfAbsent to avoid the notifyReplicatorDispatchRateLimiterConsumer + // concurrent access. + replicatorDispatchRateLimiterConsumerMap.compute(key, (__, old) -> { + if (old == null) { + old = ConcurrentHashMap.newKeySet(); + } + old.add(consumer); + return old; + }); } } - protected void updateResourceGroup(org.apache.pulsar.common.policies.data.ResourceGroup rgConfig) { - this.setResourceGroupConfigParameters(rgConfig); - val pubBmc = new BytesAndMessagesCount(); - pubBmc.messages = rgConfig.getPublishRateInMsgs(); - pubBmc.bytes = rgConfig.getPublishRateInBytes(); - this.resourceGroupPublishLimiter.update(pubBmc); + public void unregisterReplicatorDispatchRateLimiter(String remoteCluster, + Consumer consumer) { + String key = getReplicatorDispatchRateLimiterKey(remoteCluster); + synchronized (this) { + consumer.accept(null); + replicatorDispatchRateLimiterConsumerMap.computeIfPresent(key, (__, old) -> { + old.remove(consumer); + return old; + }); + } } protected long getResourceGroupNumOfNSRefs() { return this.resourceGroupNamespaceRefs.size(); } + protected long getResourceGroupNumOfTopicRefs() { + return this.resourceGroupTopicRefs.size(); + } + protected long getResourceGroupNumOfTenantRefs() { return this.resourceGroupTenantRefs.size(); } @@ -171,6 +276,9 @@ protected ResourceGroupOpStatus registerUsage(String name, ResourceGroupRefTypes case Namespaces: set = this.resourceGroupNamespaceRefs; break; + case Topics: + set = this.resourceGroupTopicRefs; + break; } if (ref) { @@ -181,7 +289,8 @@ protected ResourceGroupOpStatus registerUsage(String name, ResourceGroupRefTypes set.add(name); // If this is the first ref, register with the transport manager. - if (this.resourceGroupTenantRefs.size() + this.resourceGroupNamespaceRefs.size() == 1) { + if (this.resourceGroupTenantRefs.size() + this.resourceGroupNamespaceRefs.size() + + this.resourceGroupTopicRefs.size() == 1) { if (log.isDebugEnabled()) { log.debug("registerUsage for RG={}: registering with transport-mgr", this.resourceGroupName); } @@ -196,7 +305,8 @@ protected ResourceGroupOpStatus registerUsage(String name, ResourceGroupRefTypes set.remove(name); // If this was the last ref, unregister from the transport manager. - if (this.resourceGroupTenantRefs.size() + this.resourceGroupNamespaceRefs.size() == 0) { + if (this.resourceGroupTenantRefs.size() + this.resourceGroupNamespaceRefs.size() + + this.resourceGroupTopicRefs.size() == 0) { if (log.isDebugEnabled()) { log.debug("unRegisterUsage for RG={}: un-registering from transport-mgr", this.resourceGroupName); } @@ -215,190 +325,246 @@ public String getID() { // Transport manager mandated op. public void rgFillResourceUsage(ResourceUsage resourceUsage) { - NetworkUsage p; resourceUsage.setOwner(this.getID()); - - p = resourceUsage.setPublish(); - if (!this.setUsageInMonitoredEntity(ResourceGroupMonitoringClass.Publish, p)) { - resourceUsage.clearPublish(); - } - - p = resourceUsage.setDispatch(); - if (!this.setUsageInMonitoredEntity(ResourceGroupMonitoringClass.Dispatch, p)) { - resourceUsage.clearDispatch(); - } - - // Punt storage for now. + this.setUsageInMonitoredEntity(resourceUsage); } // Transport manager mandated op. public void rgResourceUsageListener(String broker, ResourceUsage resourceUsage) { - if (resourceUsage.hasPublish()) { - this.getUsageFromMonitoredEntity(ResourceGroupMonitoringClass.Publish, resourceUsage.getPublish(), broker); - } + this.getUsageFromMonitoredEntity(resourceUsage, broker); + } - if (resourceUsage.hasDispatch()) { - this.getUsageFromMonitoredEntity(ResourceGroupMonitoringClass.Dispatch, resourceUsage.getDispatch(), - broker); - } - // Punt storage for now. + private MonitoringKey getMonitoringKey(ResourceGroupMonitoringClass monClass, String remoteCluster) { + return new MonitoringKey(monClass, remoteCluster); } - protected BytesAndMessagesCount getConfLimits(ResourceGroupMonitoringClass monClass) throws PulsarAdminException { + protected Map getConfLimits(ResourceGroupMonitoringClass monClass) + throws PulsarAdminException { this.checkMonitoringClass(monClass); - BytesAndMessagesCount retval = new BytesAndMessagesCount(); - final PerMonitoringClassFields monEntity = this.monitoringClassFields[monClass.ordinal()]; - monEntity.localUsageStatsLock.lock(); - try { - retval.bytes = monEntity.configValuesPerPeriod.bytes; - retval.messages = monEntity.configValuesPerPeriod.messages; - } finally { - monEntity.localUsageStatsLock.unlock(); - } - return retval; + Map retStats = new HashMap<>(); + monitoringClassFieldsMap.forEach(((key, monEntity) -> { + if (key.getResourceGroupMonitoringClass() != monClass) { + return; + } + BytesAndMessagesCount bytesAndMessagesCount = + retStats.computeIfAbsent(key.getRemoteCluster(), k -> new BytesAndMessagesCount()); + monEntity.localUsageStatsLock.lock(); + try { + bytesAndMessagesCount.bytes += monEntity.configValuesPerPeriod.bytes; + bytesAndMessagesCount.messages += monEntity.configValuesPerPeriod.messages; + } finally { + monEntity.localUsageStatsLock.unlock(); + } + })); + return retStats; } - protected void incrementLocalUsageStats(ResourceGroupMonitoringClass monClass, BytesAndMessagesCount stats) - throws PulsarAdminException { - this.checkMonitoringClass(monClass); - final PerMonitoringClassFields monEntity = this.monitoringClassFields[monClass.ordinal()]; - monEntity.localUsageStatsLock.lock(); - try { - monEntity.usedLocallySinceLastReport.bytes += stats.bytes; - monEntity.usedLocallySinceLastReport.messages += stats.messages; - } finally { - monEntity.localUsageStatsLock.unlock(); + private PerMonitoringClassFields createPerMonitoringClassFields(ResourceGroupMonitoringClass monClass, + String remoteCluster) { + PerMonitoringClassFields value = + PerMonitoringClassFields.create(getCacheDuration()); + switch (monClass) { + case Publish: + value.configValuesPerPeriod.bytes = rgConfig.getPublishRateInBytes() == null + ? -1 : rgConfig.getPublishRateInBytes(); + value.configValuesPerPeriod.messages = rgConfig.getPublishRateInMsgs() == null + ? -1 : rgConfig.getPublishRateInMsgs(); + break; + case Dispatch: + value.configValuesPerPeriod.bytes = rgConfig.getDispatchRateInBytes() == null + ? -1 : rgConfig.getDispatchRateInBytes(); + value.configValuesPerPeriod.messages = rgConfig.getDispatchRateInMsgs() == null + ? -1 : rgConfig.getDispatchRateInMsgs(); + break; + case ReplicationDispatch: + requireNonNull(remoteCluster, "remoteCluster cannot be null when monClass is ReplicationDispatch"); + DispatchRate dispatchRate = + rgConfig.getReplicatorDispatchRate().get(getReplicatorDispatchRateLimiterKey(remoteCluster)); + if (dispatchRate != null) { + value.configValuesPerPeriod.bytes = dispatchRate.getDispatchThrottlingRateInByte(); + value.configValuesPerPeriod.messages = + dispatchRate.getDispatchThrottlingRateInMsg(); + } else { + value.configValuesPerPeriod.bytes = rgConfig.getReplicationDispatchRateInBytes() == null ? -1 : + rgConfig.getReplicationDispatchRateInBytes(); + value.configValuesPerPeriod.messages = rgConfig.getReplicationDispatchRateInMsgs() == null ? -1 : + rgConfig.getReplicationDispatchRateInMsgs(); + } + break; } + return value; } - protected BytesAndMessagesCount getLocalUsageStats(ResourceGroupMonitoringClass monClass) - throws PulsarAdminException { - this.checkMonitoringClass(monClass); - BytesAndMessagesCount retval = new BytesAndMessagesCount(); - final PerMonitoringClassFields monEntity = this.monitoringClassFields[monClass.ordinal()]; - monEntity.localUsageStatsLock.lock(); - try { - retval.bytes = monEntity.usedLocallySinceLastReport.bytes; - retval.messages = monEntity.usedLocallySinceLastReport.messages; - } finally { - monEntity.localUsageStatsLock.unlock(); - } - - return retval; + protected PerMonitoringClassFields getPerMonitoringClassFields(ResourceGroupMonitoringClass monClass, + String remoteCluster) { + return monitoringClassFieldsMap.computeIfAbsent(getMonitoringKey(monClass, remoteCluster), + (__) -> createPerMonitoringClassFields(monClass, remoteCluster)); } - protected BytesAndMessagesCount getLocalUsageStatsCumulative(ResourceGroupMonitoringClass monClass) + protected void incrementLocalUsageStats(ResourceGroupMonitoringClass monClass, BytesAndMessagesCount stats, + String remoteCluster) throws PulsarAdminException { this.checkMonitoringClass(monClass); - BytesAndMessagesCount retval = new BytesAndMessagesCount(); - final PerMonitoringClassFields monEntity = this.monitoringClassFields[monClass.ordinal()]; - monEntity.localUsageStatsLock.lock(); - try { - // If the total wasn't accumulated yet (i.e., a report wasn't sent yet), just return the - // partial accumulation in usedLocallySinceLastReport. - if (monEntity.totalUsedLocally.messages == 0) { - retval.bytes = monEntity.usedLocallySinceLastReport.bytes; - retval.messages = monEntity.usedLocallySinceLastReport.messages; - } else { - retval.bytes = monEntity.totalUsedLocally.bytes; - retval.messages = monEntity.totalUsedLocally.messages; + synchronized (this) { + PerMonitoringClassFields monEntity = getPerMonitoringClassFields(monClass, remoteCluster); + monEntity.localUsageStatsLock.lock(); + try { + monEntity.usedLocallySinceLastReport.bytes += stats.bytes; + monEntity.usedLocallySinceLastReport.messages += stats.messages; + } finally { + monEntity.localUsageStatsLock.unlock(); } - } finally { - monEntity.localUsageStatsLock.unlock(); } - - return retval; } - protected BytesAndMessagesCount getLocalUsageStatsFromBrokerReports(ResourceGroupMonitoringClass monClass) + protected Map getLocalUsageStatsCumulative(ResourceGroupMonitoringClass monClass) throws PulsarAdminException { this.checkMonitoringClass(monClass); - val retval = new BytesAndMessagesCount(); - final PerMonitoringClassFields monEntity = this.monitoringClassFields[monClass.ordinal()]; - String myBrokerId = this.rgs.getPulsar().getBrokerServiceUrl(); - PerBrokerUsageStats pbus; - - monEntity.usageFromOtherBrokersLock.lock(); - try { - pbus = monEntity.usageFromOtherBrokers.get(myBrokerId); - } finally { - monEntity.usageFromOtherBrokersLock.unlock(); - } - - if (pbus != null) { - retval.bytes = pbus.usedValues.bytes; - retval.messages = pbus.usedValues.messages; - } else { - if (log.isDebugEnabled()) { - log.debug("getLocalUsageStatsFromBrokerReports: no usage report found for broker={} and monClass={}", - myBrokerId, monClass); - } + Map retval = new HashMap<>(); + synchronized (this) { + monitoringClassFieldsMap.forEach((key, monEntity) -> { + BytesAndMessagesCount bytesAndMessagesCount = + retval.computeIfAbsent(key.getRemoteCluster(), k -> new BytesAndMessagesCount()); + monEntity.localUsageStatsLock.lock(); + try { + // If the total wasn't accumulated yet (i.e., a report wasn't sent yet), just return the + // partial accumulation in usedLocallySinceLastReport. + if (monEntity.totalUsedLocally.messages == 0) { + bytesAndMessagesCount.bytes = monEntity.usedLocallySinceLastReport.bytes; + bytesAndMessagesCount.messages = monEntity.usedLocallySinceLastReport.messages; + } else { + bytesAndMessagesCount.bytes = monEntity.totalUsedLocally.bytes; + bytesAndMessagesCount.messages = monEntity.totalUsedLocally.messages; + } + } finally { + monEntity.localUsageStatsLock.unlock(); + } + }); } - return retval; } - protected BytesAndMessagesCount getGlobalUsageStats(ResourceGroupMonitoringClass monClass) - throws PulsarAdminException { + protected Map getLocalUsageStatsFromBrokerReports( + ResourceGroupMonitoringClass monClass) + throws PulsarAdminException { this.checkMonitoringClass(monClass); - final PerMonitoringClassFields monEntity = this.monitoringClassFields[monClass.ordinal()]; - monEntity.usageFromOtherBrokersLock.lock(); - BytesAndMessagesCount retStats = new BytesAndMessagesCount(); - try { - monEntity.usageFromOtherBrokers.forEach((broker, brokerUsage) -> { - retStats.bytes += brokerUsage.usedValues.bytes; - retStats.messages += brokerUsage.usedValues.messages; - }); - } finally { - monEntity.usageFromOtherBrokersLock.unlock(); - } + String myBrokerId = this.rgs.getPulsar().getBrokerServiceUrl(); + Map retStats = new HashMap<>(); + monitoringClassFieldsMap.forEach(((key, monEntity) -> { + if (key.getResourceGroupMonitoringClass() != monClass) { + return; + } + BytesAndMessagesCount bytesAndMessagesCount = + retStats.computeIfAbsent(key.getRemoteCluster(), k -> new BytesAndMessagesCount()); + monEntity.usageFromOtherBrokersLock.lock(); + try { + PerBrokerUsageStats pbus = monEntity.usageFromOtherBrokers.getIfPresent(myBrokerId); + if (pbus != null) { + bytesAndMessagesCount.bytes += pbus.usedValues.bytes; + bytesAndMessagesCount.messages += pbus.usedValues.messages; + } + } finally { + monEntity.usageFromOtherBrokersLock.unlock(); + } + })); + if (retStats.isEmpty()) { + log.info("getLocalUsageStatsFromBrokerReports: no usage report found for broker={} and monClass={}", + myBrokerId, monClass); + } return retStats; } - protected BytesAndMessagesCount updateLocalQuota(ResourceGroupMonitoringClass monClass, - BytesAndMessagesCount newQuota) throws PulsarAdminException { - // Only the Publish side is functional now; add the Dispatch side code when the consume side is ready. - if (!ResourceGroupMonitoringClass.Publish.equals(monClass)) { - if (log.isDebugEnabled()) { - log.debug("Doing nothing for monClass={}; only Publish is functional", monClass); + protected Map getGlobalUsageStats(ResourceGroupMonitoringClass monClass) + throws PulsarAdminException { + this.checkMonitoringClass(monClass); + + Map retStats = new HashMap<>(); + monitoringClassFieldsMap.forEach(((key, monEntity) -> { + if (key.getResourceGroupMonitoringClass() != monClass) { + return; } - return null; - } + BytesAndMessagesCount bytesAndMessagesCount = + retStats.computeIfAbsent(key.getRemoteCluster(), k -> new BytesAndMessagesCount()); + monEntity.usageFromOtherBrokersLock.lock(); + try { + monEntity.usageFromOtherBrokers.asMap().forEach((broker, brokerUsage) -> { + bytesAndMessagesCount.bytes += brokerUsage.usedValues.bytes; + bytesAndMessagesCount.messages += brokerUsage.usedValues.messages; + }); + } finally { + monEntity.usageFromOtherBrokersLock.unlock(); + } + })); + return retStats; + } + protected void updateLocalQuota(ResourceGroupMonitoringClass monClass, + BytesAndMessagesCount newQuota, String remoteCluster) + throws PulsarAdminException { this.checkMonitoringClass(monClass); - BytesAndMessagesCount oldBMCount; - - final PerMonitoringClassFields monEntity = this.monitoringClassFields[monClass.ordinal()]; - monEntity.localUsageStatsLock.lock(); - oldBMCount = monEntity.quotaForNextPeriod; - try { - monEntity.quotaForNextPeriod = newQuota; - this.resourceGroupPublishLimiter.update(newQuota); - } finally { - monEntity.localUsageStatsLock.unlock(); - } - if (log.isDebugEnabled()) { - log.debug("updateLocalQuota for RG={}: set local {} quota to bytes={}, messages={}", - this.resourceGroupName, monClass, newQuota.bytes, newQuota.messages); - } - return oldBMCount; + synchronized (this) { + if (isClosed) { + return; + } + monitoringClassFieldsMap.forEach((key, monEntity) -> { + if (!key.getResourceGroupMonitoringClass().equals(monClass)) { + return; + } + monEntity.localUsageStatsLock.lock(); + try { + switch (monClass) { + case ReplicationDispatch: + String replicatorDispatchRateLimiterKey = + getReplicatorDispatchRateLimiterKey(key.getRemoteCluster()); + ResourceGroupDispatchLimiter limiter = null; + if (remoteCluster == null) { + // global replication dispatch rate limiter. + limiter = resourceGroupReplicationDispatchLimiter; + } else { + if (Objects.equals(remoteCluster, key.getRemoteCluster())) { + limiter = + this.replicatorDispatchRateLimiterMap.get(replicatorDispatchRateLimiterKey); + if (limiter == null) { + // Limiter was not found, which will lazily load. + return; + } + } + } + + if (limiter != null) { + ResourceGroupRateLimiterManager.updateReplicationDispatchRateLimiter(limiter, newQuota); + } + break; + case Publish: + this.resourceGroupPublishLimiter.update(newQuota); + break; + case Dispatch: + ResourceGroupRateLimiterManager.updateDispatchRateLimiter(resourceGroupDispatchLimiter, + newQuota); + break; + default: + if (log.isDebugEnabled()) { + log.debug("Doing nothing for monClass={};", monClass); + } + } + } finally { + monEntity.localUsageStatsLock.unlock(); + } + if (log.isDebugEnabled()) { + log.debug("updateLocalQuota for RG={}: set local {} quota to bytes={}, messages={}", + this.resourceGroupName, monClass, newQuota.bytes, newQuota.messages); + } + }); + } } protected BytesAndMessagesCount getRgPublishRateLimiterValues() { BytesAndMessagesCount retVal; - final PerMonitoringClassFields monEntity = - this.monitoringClassFields[ResourceGroupMonitoringClass.Publish.ordinal()]; - monEntity.localUsageStatsLock.lock(); - try { - retVal = this.resourceGroupPublishLimiter.getResourceGroupPublishValues(); - } finally { - monEntity.localUsageStatsLock.unlock(); - } - + retVal = this.resourceGroupPublishLimiter.getResourceGroupPublishValues(); return retVal; } @@ -428,9 +594,16 @@ protected static BytesAndMessagesCount accumulateBMCount(BytesAndMessagesCount . } private void checkMonitoringClass(ResourceGroupMonitoringClass monClass) throws PulsarAdminException { - if (monClass != ResourceGroupMonitoringClass.Publish && monClass != ResourceGroupMonitoringClass.Dispatch) { - String errMesg = "Unexpected monitoring class: " + monClass; - throw new PulsarAdminException(errMesg); + switch (monClass) { + case Publish: + break; + case Dispatch: + break; + case ReplicationDispatch: + break; + default: + String errMesg = "Unexpected monitoring class: " + monClass; + throw new PulsarAdminException(errMesg); } } @@ -438,140 +611,197 @@ private void checkMonitoringClass(ResourceGroupMonitoringClass monClass) throws // for reporting local stats to other brokers. // Returns true if something was filled. // Visibility for unit testing. - protected boolean setUsageInMonitoredEntity(ResourceGroupMonitoringClass monClass, NetworkUsage p) { - long bytesUsed, messagesUsed; - boolean sendReport; - int numSuppressions = 0; - PerMonitoringClassFields monEntity; - - final int idx = monClass.ordinal(); - monEntity = this.monitoringClassFields[idx]; - - monEntity.localUsageStatsLock.lock(); - try { - sendReport = this.rgs.quotaCalculator.needToReportLocalUsage( - monEntity.usedLocallySinceLastReport.bytes, - monEntity.lastReportedValues.bytes, - monEntity.usedLocallySinceLastReport.messages, - monEntity.lastReportedValues.messages, - monEntity.lastResourceUsageFillTimeMSecsSinceEpoch); - - bytesUsed = monEntity.usedLocallySinceLastReport.bytes; - messagesUsed = monEntity.usedLocallySinceLastReport.messages; - monEntity.usedLocallySinceLastReport.bytes = monEntity.usedLocallySinceLastReport.messages = 0; - if (sendReport) { - p.setBytesPerPeriod(bytesUsed); - p.setMessagesPerPeriod(messagesUsed); - monEntity.lastReportedValues.bytes = bytesUsed; - monEntity.lastReportedValues.messages = messagesUsed; - monEntity.numSuppressedUsageReports = 0; - monEntity.totalUsedLocally.bytes += bytesUsed; - monEntity.totalUsedLocally.messages += messagesUsed; - monEntity.lastResourceUsageFillTimeMSecsSinceEpoch = System.currentTimeMillis(); - } else { - numSuppressions = monEntity.numSuppressedUsageReports++; + protected void setUsageInMonitoredEntity(ResourceUsage resourceUsage) { + monitoringClassFieldsMap.forEach((key, monEntity) -> { + monEntity.localUsageStatsLock.lock(); + try { + boolean sendReport = this.rgs.quotaCalculator.needToReportLocalUsage( + monEntity.usedLocallySinceLastReport.bytes, + monEntity.lastReportedValues.bytes, + monEntity.usedLocallySinceLastReport.messages, + monEntity.lastReportedValues.messages, + monEntity.lastResourceUsageFillTimeMSecsSinceEpoch); + + long bytesUsed = monEntity.usedLocallySinceLastReport.bytes; + long messagesUsed = monEntity.usedLocallySinceLastReport.messages; + monEntity.usedLocallySinceLastReport.bytes = monEntity.usedLocallySinceLastReport.messages = 0; + int numSuppressions = 0; + if (sendReport) { + switch (key.getResourceGroupMonitoringClass()) { + case Publish: + NetworkUsage publish = resourceUsage.setPublish(); + publish.setMessagesPerPeriod(messagesUsed); + publish.setBytesPerPeriod(bytesUsed); + break; + case Dispatch: + NetworkUsage dispatch = resourceUsage.setDispatch(); + dispatch.setMessagesPerPeriod(messagesUsed); + dispatch.setBytesPerPeriod(bytesUsed); + break; + case ReplicationDispatch: + ReplicatorUsage replicationDispatch = resourceUsage.addReplicator(); + replicationDispatch.setLocalCluster(rgs.getPulsar().getConfiguration().getClusterName()); + replicationDispatch.setRemoteCluster(key.getRemoteCluster()); + NetworkUsage networkUsage = replicationDispatch.setNetworkUsage(); + networkUsage.setMessagesPerPeriod(messagesUsed); + networkUsage.setBytesPerPeriod(bytesUsed); + break; + } + monEntity.lastReportedValues.bytes = bytesUsed; + monEntity.lastReportedValues.messages = messagesUsed; + monEntity.numSuppressedUsageReports = 0; + monEntity.lastResourceUsageFillTimeMSecsSinceEpoch = System.currentTimeMillis(); + } else { + numSuppressions = monEntity.numSuppressedUsageReports++; + } + + final String rgName = this.ruPublisher != null ? this.ruPublisher.getID() : this.resourceGroupName; + double sentCount = sendReport ? 1 : 0; + rgLocalUsageReportCount.labels(rgName, key.getResourceGroupMonitoringClass().name()).inc(sentCount); + if (sendReport) { + if (log.isDebugEnabled()) { + log.debug("fillResourceUsage for RG={}: filled a {} update; bytes={}, messages={}", + rgName, key.getResourceGroupMonitoringClass(), bytesUsed, messagesUsed); + } + } else { + if (log.isDebugEnabled()) { + log.debug("fillResourceUsage for RG={}: report for {} suppressed " + + "(suppressions={} since last sent report)", + rgName, key.getResourceGroupMonitoringClass(), numSuppressions); + } + } + } finally { + monEntity.localUsageStatsLock.unlock(); } + }); + } - } finally { - monEntity.localUsageStatsLock.unlock(); - } - final String rgName = this.ruPublisher != null ? this.ruPublisher.getID() : this.resourceGroupName; - double sentCount = sendReport ? 1 : 0; - rgLocalUsageReportCount.labels(rgName, monClass.name()).inc(sentCount); - if (sendReport) { - if (log.isDebugEnabled()) { - log.debug("fillResourceUsage for RG={}: filled a {} update; bytes={}, messages={}", - rgName, monClass, bytesUsed, messagesUsed); - } - } else { - if (log.isDebugEnabled()) { - log.debug("fillResourceUsage for RG={}: report for {} suppressed " - + "(suppressions={} since last sent report)", - rgName, monClass, numSuppressions); + private void updateUsageFromOtherBrokers(MonitoringKey key, Consumer consumer) { + PerMonitoringClassFields monEntity = monitoringClassFieldsMap.computeIfAbsent(key, (__)->{ + synchronized (this) { + return createPerMonitoringClassFields(key.getResourceGroupMonitoringClass(), key.getRemoteCluster()); } - } - - return sendReport; + }); + consumer.accept(monEntity); } // Update fields in a particular monitoring class from a given broker in the // transport-manager callback for listening to usage reports. - private void getUsageFromMonitoredEntity(ResourceGroupMonitoringClass monClass, NetworkUsage p, String broker) { - final int idx = monClass.ordinal(); - PerMonitoringClassFields monEntity; - PerBrokerUsageStats usageStats, oldUsageStats; - long oldByteCount, oldMessageCount; - long newByteCount, newMessageCount; - - monEntity = this.monitoringClassFields[idx]; - usageStats = monEntity.usageFromOtherBrokers.get(broker); - if (usageStats == null) { - usageStats = new PerBrokerUsageStats(); - usageStats.usedValues = new BytesAndMessagesCount(); - } - monEntity.usageFromOtherBrokersLock.lock(); - try { - newByteCount = p.getBytesPerPeriod(); - usageStats.usedValues.bytes = newByteCount; - newMessageCount = p.getMessagesPerPeriod(); - usageStats.usedValues.messages = newMessageCount; - usageStats.lastResourceUsageReadTimeMSecsSinceEpoch = System.currentTimeMillis(); - oldUsageStats = monEntity.usageFromOtherBrokers.put(broker, usageStats); - } finally { - monEntity.usageFromOtherBrokersLock.unlock(); + protected void getUsageFromMonitoredEntity(ResourceUsage resourceUsage, String broker) { + List monitoringKeyList = new LinkedList<>(); + if (resourceUsage.hasPublish()) { + monitoringKeyList.add(getMonitoringKey(ResourceGroupMonitoringClass.Publish, null)); } - rgRemoteUsageReportsBytes.labels(this.ruConsumer.getID(), monClass.name(), broker).inc(newByteCount); - rgRemoteUsageReportsMessages.labels(this.ruConsumer.getID(), monClass.name(), broker).inc(newMessageCount); - - oldByteCount = oldMessageCount = -1; - if (oldUsageStats != null) { - oldByteCount = oldUsageStats.usedValues.bytes; - oldMessageCount = oldUsageStats.usedValues.messages; + if (resourceUsage.hasDispatch()) { + monitoringKeyList.add(getMonitoringKey(ResourceGroupMonitoringClass.Dispatch, null)); } - - if (log.isDebugEnabled()) { - log.debug("resourceUsageListener for RG={}: updated {} stats for broker={} " - + "with bytes={} (old ={}), messages={} (old={})", - this.resourceGroupName, monClass, broker, - newByteCount, oldByteCount, - newMessageCount, oldMessageCount); + if (resourceUsage.getReplicatorsCount() != 0) { + resourceUsage.getReplicatorsList().forEach(replicator -> { + monitoringKeyList.add(getMonitoringKey(ResourceGroupMonitoringClass.ReplicationDispatch, + replicator.getRemoteCluster())); + }); } - } - private void setResourceGroupMonitoringClassFields() { - PerMonitoringClassFields monClassFields; - for (int idx = 0; idx < ResourceGroupMonitoringClass.values().length; idx++) { - this.monitoringClassFields[idx] = new PerMonitoringClassFields(); - - monClassFields = this.monitoringClassFields[idx]; - monClassFields.configValuesPerPeriod = new BytesAndMessagesCount(); - monClassFields.usedLocallySinceLastReport = new BytesAndMessagesCount(); - monClassFields.lastReportedValues = new BytesAndMessagesCount(); - monClassFields.quotaForNextPeriod = new BytesAndMessagesCount(); - monClassFields.totalUsedLocally = new BytesAndMessagesCount(); - monClassFields.usageFromOtherBrokers = new HashMap<>(); - - monClassFields.usageFromOtherBrokersLock = new ReentrantLock(); - // ToDo: Change the following to a ReadWrite lock if needed. - monClassFields.localUsageStatsLock = new ReentrantLock(); - } - } - - private void setResourceGroupConfigParameters(org.apache.pulsar.common.policies.data.ResourceGroup rgConfig) { - int idx; - - idx = ResourceGroupMonitoringClass.Publish.ordinal(); - this.monitoringClassFields[idx].configValuesPerPeriod.bytes = rgConfig.getPublishRateInBytes() == null - ? -1 : rgConfig.getPublishRateInBytes(); - this.monitoringClassFields[idx].configValuesPerPeriod.messages = rgConfig.getPublishRateInMsgs() == null - ? -1 : rgConfig.getPublishRateInMsgs(); - - idx = ResourceGroupMonitoringClass.Dispatch.ordinal(); - this.monitoringClassFields[idx].configValuesPerPeriod.bytes = rgConfig.getDispatchRateInBytes() == null - ? -1 : rgConfig.getDispatchRateInBytes(); - this.monitoringClassFields[idx].configValuesPerPeriod.messages = rgConfig.getDispatchRateInMsgs() == null - ? -1 : rgConfig.getDispatchRateInMsgs(); + monitoringKeyList.forEach((key) -> updateUsageFromOtherBrokers(key, monEntity -> { + monEntity.usageFromOtherBrokersLock.lock(); + PerBrokerUsageStats usageStats = monEntity.usageFromOtherBrokers.get(broker, (__) -> { + PerBrokerUsageStats perBrokerUsageStats = new PerBrokerUsageStats(); + perBrokerUsageStats.usedValues = new BytesAndMessagesCount(); + return perBrokerUsageStats; + }); + assert usageStats != null; + try { + NetworkUsage p = null; + switch (key.getResourceGroupMonitoringClass()) { + case Publish: + p = resourceUsage.hasPublish() ? resourceUsage.getPublish() : null; + break; + case Dispatch: + p = resourceUsage.hasDispatch() ? resourceUsage.getDispatch() : null; + break; + case ReplicationDispatch: + ReplicatorUsage replicatorUsage = resourceUsage.getReplicatorsList().stream() + .filter(n -> Objects.equals(n.getRemoteCluster(), key.getRemoteCluster())) + .findFirst() + .orElse(null); + if (replicatorUsage != null) { + p = replicatorUsage.getNetworkUsage(); + } + break; + } + if (p == null) { + return; + } + long newByteCount = p.getBytesPerPeriod(); + usageStats.usedValues.bytes = newByteCount; + long newMessageCount = p.getMessagesPerPeriod(); + usageStats.usedValues.messages = newMessageCount; + usageStats.lastResourceUsageReadTimeMSecsSinceEpoch = System.currentTimeMillis(); + String remoteCluster = key.getRemoteCluster() != null ? key.getRemoteCluster() : ""; + rgRemoteUsageReportsBytes.labels(this.ruConsumer.getID(), + ResourceGroupMonitoringClass.Publish.name(), broker, remoteCluster) + .inc(newByteCount); + rgRemoteUsageReportsMessages.labels(this.ruConsumer.getID(), + ResourceGroupMonitoringClass.Publish.name(), broker, remoteCluster) + .inc(newMessageCount); + rgRemoteUsageReportsBytesGauge.labels(this.ruConsumer.getID(), + ResourceGroupMonitoringClass.Publish.name(), broker, remoteCluster) + .set(newByteCount); + rgRemoteUsageReportsMessagesGauge.labels(this.ruConsumer.getID(), + ResourceGroupMonitoringClass.Publish.name(), broker, remoteCluster) + .set(newMessageCount); + } finally { + monEntity.usageFromOtherBrokersLock.unlock(); + } + })); + } + + private long getCacheDuration() { + ServiceConfiguration conf = rgs.getPulsar().getConfiguration(); + long resourceUsageTransportPublishIntervalInSecs = conf.getResourceUsageTransportPublishIntervalInSecs(); + int maxUsageReportSuppressRounds = Math.max(conf.getResourceUsageMaxUsageReportSuppressRounds(), 1); + // Usage report data is cached to the memory, when the broker is restart or offline, we need an elimination + // strategy to release the quota occupied by other broker. + // + // Considering that each broker starts at a different time, the cache time should be equal to the mandatory + // reporting period * 2. + return TimeUnit.SECONDS.toMillis( + resourceUsageTransportPublishIntervalInSecs * maxUsageReportSuppressRounds * 2); + } + + private void updateMonitoringClassFieldsMap(org.apache.pulsar.common.policies.data.ResourceGroup rgConfig) { + this.rgConfig = rgConfig; + monitoringClassFieldsMap.forEach((key, monEntity) -> { + switch (key.getResourceGroupMonitoringClass()) { + case Publish: + monEntity.configValuesPerPeriod.bytes = rgConfig.getPublishRateInBytes() == null + ? -1 : rgConfig.getPublishRateInBytes(); + monEntity.configValuesPerPeriod.messages = rgConfig.getPublishRateInMsgs() == null + ? -1 : rgConfig.getPublishRateInMsgs(); + break; + case Dispatch: + monEntity.configValuesPerPeriod.bytes = rgConfig.getDispatchRateInBytes() == null + ? -1 : rgConfig.getDispatchRateInBytes(); + monEntity.configValuesPerPeriod.messages = rgConfig.getDispatchRateInMsgs() == null + ? -1 : rgConfig.getDispatchRateInMsgs(); + break; + case ReplicationDispatch: + requireNonNull(key.getRemoteCluster(), + "remoteCluster cannot be null when monClass is ReplicationDispatch"); + DispatchRate dispatchRate = rgConfig.getReplicatorDispatchRate() + .get(getReplicatorDispatchRateLimiterKey(key.getRemoteCluster())); + if (dispatchRate != null) { + monEntity.configValuesPerPeriod.bytes = dispatchRate.getDispatchThrottlingRateInByte(); + monEntity.configValuesPerPeriod.messages = + dispatchRate.getDispatchThrottlingRateInMsg(); + } else { + monEntity.configValuesPerPeriod.bytes = rgConfig.getReplicationDispatchRateInBytes(); + monEntity.configValuesPerPeriod.messages = rgConfig.getReplicationDispatchRateInMsgs(); + } + break; + } + }); } private void setDefaultResourceUsageTransportHandlers() { @@ -601,14 +831,45 @@ public void acceptResourceUsage(String broker, ResourceUsage resourceUsage) { } @VisibleForTesting - PerMonitoringClassFields getMonitoredEntity(ResourceGroupMonitoringClass monClass) { - return this.monitoringClassFields[monClass.ordinal()]; + PerMonitoringClassFields getMonitoredEntity(ResourceGroupMonitoringClass monClass, String remoteCluster) { + return getPerMonitoringClassFields(monClass, remoteCluster); } public final String resourceGroupName; - public PerMonitoringClassFields[] monitoringClassFields = - new PerMonitoringClassFields[ResourceGroupMonitoringClass.values().length]; + @ToString + public static class MonitoringKey { + @Getter + private final ResourceGroupMonitoringClass resourceGroupMonitoringClass; + @Getter + private final String remoteCluster; + + public MonitoringKey(ResourceGroupMonitoringClass resourceGroupMonitoringClass, String remoteCluster) { + this.resourceGroupMonitoringClass = resourceGroupMonitoringClass; + this.remoteCluster = remoteCluster; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof MonitoringKey)) { + return false; + } + MonitoringKey that = (MonitoringKey) o; + return Objects.equals(resourceGroupMonitoringClass, that.resourceGroupMonitoringClass) + && Objects.equals(remoteCluster, that.remoteCluster); + } + + @Override + public int hashCode() { + return Objects.hash(resourceGroupMonitoringClass, remoteCluster); + } + } + + @Getter + public final Map monitoringClassFieldsMap = new ConcurrentHashMap<>(); private static final Logger log = LoggerFactory.getLogger(ResourceGroupService.class); @@ -617,6 +878,7 @@ PerMonitoringClassFields getMonitoredEntity(ResourceGroupMonitoringClass monClas // across all of its usage classes (publish/dispatch/...). private Set resourceGroupTenantRefs = ConcurrentHashMap.newKeySet(); private Set resourceGroupNamespaceRefs = ConcurrentHashMap.newKeySet(); + private Set resourceGroupTopicRefs = ConcurrentHashMap.newKeySet(); // Blobs required for transport manager's resource-usage register/unregister ops. ResourceUsageConsumer ruConsumer; @@ -628,18 +890,28 @@ PerMonitoringClassFields getMonitoredEntity(ResourceGroupMonitoringClass monClas // Labels for the various counters used here. private static final String[] resourceGroupMontoringclassLabels = {"ResourceGroup", "MonitoringClass"}; private static final String[] resourceGroupMontoringclassRemotebrokerLabels = - {"ResourceGroup", "MonitoringClass", "RemoteBroker"}; + {"ResourceGroup", "MonitoringClass", "RemoteBroker", "RemoteBrokerName"}; private static final Counter rgRemoteUsageReportsBytes = Counter.build() .name("pulsar_resource_group_remote_usage_bytes_used") .help("Bytes used reported about this from a remote broker") .labelNames(resourceGroupMontoringclassRemotebrokerLabels) .register(); + private static final Gauge rgRemoteUsageReportsBytesGauge = Gauge.build() + .name("pulsar_resource_group_remote_usage_bytes_used_gauge") + .help("Bytes used reported about this from a remote broker") + .labelNames(resourceGroupMontoringclassRemotebrokerLabels) + .register(); private static final Counter rgRemoteUsageReportsMessages = Counter.build() .name("pulsar_resource_group_remote_usage_messages_used") .help("Messages used reported about this from a remote broker") .labelNames(resourceGroupMontoringclassRemotebrokerLabels) .register(); + private static final Gauge rgRemoteUsageReportsMessagesGauge = Gauge.build() + .name("pulsar_resource_group_remote_usage_messages_used_gauge") + .help("Messages used reported about this from a remote broker") + .labelNames(resourceGroupMontoringclassRemotebrokerLabels) + .register(); private static final Counter rgLocalUsageReportCount = Counter.build() .name("pulsar_resource_group_local_usage_reported") @@ -647,10 +919,26 @@ PerMonitoringClassFields getMonitoredEntity(ResourceGroupMonitoringClass monClas .labelNames(resourceGroupMontoringclassLabels) .register(); + @Getter + private org.apache.pulsar.common.policies.data.ResourceGroup rgConfig; + + private final Object replicatorDispatchRateLock = new Object(); + // Publish rate limiter for the resource group @Getter protected ResourceGroupPublishLimiter resourceGroupPublishLimiter; + @Getter + private final ResourceGroupDispatchLimiter resourceGroupReplicationDispatchLimiter; + + private Map replicatorDispatchRateLimiterMap = + new ConcurrentHashMap<>(); + private Map>> replicatorDispatchRateLimiterConsumerMap = + new ConcurrentHashMap<>(); + + @Getter + protected ResourceGroupDispatchLimiter resourceGroupDispatchLimiter; + protected static class PerMonitoringClassFields { // This lock covers all the "local" counts (i.e., except for the per-broker usage stats). Lock localUsageStatsLock; @@ -675,11 +963,33 @@ protected static class PerMonitoringClassFields { int numSuppressedUsageReports; // Accumulated stats of local usage. + @VisibleForTesting BytesAndMessagesCount totalUsedLocally; // This lock covers all the non-local usage counts, received from other brokers. Lock usageFromOtherBrokersLock; - public HashMap usageFromOtherBrokers; + public Cache usageFromOtherBrokers; + + private PerMonitoringClassFields(){ + + } + + static PerMonitoringClassFields create(long durationMs) { + PerMonitoringClassFields perMonitoringClassFields = new PerMonitoringClassFields(); + perMonitoringClassFields.configValuesPerPeriod = new BytesAndMessagesCount(); + perMonitoringClassFields.usedLocallySinceLastReport = new BytesAndMessagesCount(); + perMonitoringClassFields.lastReportedValues = new BytesAndMessagesCount(); + perMonitoringClassFields.quotaForNextPeriod = new BytesAndMessagesCount(); + perMonitoringClassFields.totalUsedLocally = new BytesAndMessagesCount(); + perMonitoringClassFields.usageFromOtherBrokersLock = new ReentrantLock(); + // ToDo: Change the following to a ReadWrite lock if needed. + perMonitoringClassFields.localUsageStatsLock = new ReentrantLock(); + + perMonitoringClassFields.usageFromOtherBrokers = Caffeine.newBuilder() + .expireAfterWrite(durationMs, TimeUnit.MILLISECONDS) + .build(); + return perMonitoringClassFields; + } } // Usage stats for this RG obtained from other brokers. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupDispatchLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupDispatchLimiter.java new file mode 100644 index 0000000000000..0796266114278 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupDispatchLimiter.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.resourcegroup; + +import java.util.concurrent.TimeUnit; +import org.apache.pulsar.broker.qos.AsyncTokenBucket; + +public class ResourceGroupDispatchLimiter implements AutoCloseable { + private static final long RATE_PERIOD_NANOS = TimeUnit.SECONDS.toNanos(1); + + private volatile AsyncTokenBucket dispatchRateLimiterOnMessage; + private volatile AsyncTokenBucket dispatchRateLimiterOnByte; + + public ResourceGroupDispatchLimiter(long dispatchRateInMsgs, long dispatchRateInBytes) { + update(dispatchRateInMsgs, dispatchRateInBytes); + } + + public void update(long dispatchRateInMsgs, long dispatchRateInBytes) { + dispatchRateLimiterOnMessage = createOrReuseTokenBucket(dispatchRateLimiterOnMessage, dispatchRateInMsgs); + dispatchRateLimiterOnByte = createOrReuseTokenBucket(dispatchRateLimiterOnByte, dispatchRateInBytes); + } + + private AsyncTokenBucket createOrReuseTokenBucket(AsyncTokenBucket currentLimiter, long rate) { + if (rate <= 0) { + return null; + } + if (currentLimiter != null && currentLimiter.getRate() == rate) { + return currentLimiter; + } + return AsyncTokenBucket.builder() + .rate(rate) + .ratePeriodNanos(RATE_PERIOD_NANOS) + .addTokensResolutionNanos(RATE_PERIOD_NANOS) + .build(); + } + + /** + * returns available msg-permit if msg-dispatch-throttling is enabled else it returns -1. + * + * @return + */ + public long getAvailableDispatchRateLimitOnMsg() { + AsyncTokenBucket localDispatchRateLimiterOnMessage = dispatchRateLimiterOnMessage; + return localDispatchRateLimiterOnMessage == null ? -1 + : Math.max(localDispatchRateLimiterOnMessage.getTokens(), 0); + } + + /** + * returns available byte-permit if msg-dispatch-throttling is enabled else it returns -1. + * + * @return + */ + public long getAvailableDispatchRateLimitOnByte() { + AsyncTokenBucket localDispatchRateLimiterOnByte = dispatchRateLimiterOnByte; + return localDispatchRateLimiterOnByte == null ? -1 + : Math.max(localDispatchRateLimiterOnByte.getTokens(), 0); + } + + /** + * It acquires msg and bytes permits from rate-limiter and returns if acquired permits succeed. + * + * @param numberOfMessages + * @param byteSize + */ + public void consumeDispatchQuota(long numberOfMessages, long byteSize) { + AsyncTokenBucket localDispatchRateLimiterOnMessage = dispatchRateLimiterOnMessage; + if (numberOfMessages > 0 && localDispatchRateLimiterOnMessage != null) { + localDispatchRateLimiterOnMessage.consumeTokens(numberOfMessages); + } + AsyncTokenBucket localDispatchRateLimiterOnByte = dispatchRateLimiterOnByte; + if (byteSize > 0 && localDispatchRateLimiterOnByte != null) { + localDispatchRateLimiterOnByte.consumeTokens(byteSize); + } + } + + /** + * It acquires msg and bytes permits from rate-limiter and returns if acquired permits succeed. + * + * @param numberOfMessages + * @param byteSize + */ + public boolean tryAcquire(long numberOfMessages, long byteSize) { + boolean res = true; + AsyncTokenBucket localDispatchRateLimiterOnMessage = dispatchRateLimiterOnMessage; + if (numberOfMessages > 0 && localDispatchRateLimiterOnMessage != null) { + res &= !localDispatchRateLimiterOnMessage.consumeTokensAndCheckIfContainsTokens(numberOfMessages); + } + AsyncTokenBucket localDispatchRateLimiterOnByte = dispatchRateLimiterOnByte; + if (byteSize > 0 && localDispatchRateLimiterOnByte != null) { + res &= !localDispatchRateLimiterOnByte.consumeTokensAndCheckIfContainsTokens(byteSize); + } + + return res; + } + + /** + * Checks if dispatch-rate limiting is enabled. + * + * @return + */ + public boolean isDispatchRateLimitingEnabled() { + return dispatchRateLimiterOnMessage != null || dispatchRateLimiterOnByte != null; + } + + public void close() { + dispatchRateLimiterOnMessage = null; + dispatchRateLimiterOnByte = null; + } + + /** + * Get configured msg dispatch-throttling rate. Returns -1 if not configured + * + * @return + */ + public long getDispatchRateOnMsg() { + AsyncTokenBucket localDispatchRateLimiterOnMessage = dispatchRateLimiterOnMessage; + return localDispatchRateLimiterOnMessage != null ? localDispatchRateLimiterOnMessage.getRate() : -1; + } + + /** + * Get configured byte dispatch-throttling rate. Returns -1 if not configured + * + * @return + */ + public long getDispatchRateOnByte() { + AsyncTokenBucket localDispatchRateLimiterOnByte = dispatchRateLimiterOnByte; + return localDispatchRateLimiterOnByte != null ? localDispatchRateLimiterOnByte.getRate() : -1; + } + + +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManager.java new file mode 100644 index 0000000000000..e55f35ae2df7d --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManager.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.resourcegroup; + +import java.util.Optional; +import org.apache.pulsar.broker.resourcegroup.ResourceGroup.BytesAndMessagesCount; + +public class ResourceGroupRateLimiterManager { + + static ResourceGroupDispatchLimiter newReplicationDispatchRateLimiter( + org.apache.pulsar.common.policies.data.ResourceGroup resourceGroup) { + long msgs = Optional.ofNullable(resourceGroup.getReplicationDispatchRateInMsgs()).orElse(-1L); + long bytes = Optional.ofNullable(resourceGroup.getReplicationDispatchRateInBytes()).orElse(-1L); + return newReplicationDispatchRateLimiter(msgs, bytes); + } + + static ResourceGroupDispatchLimiter newReplicationDispatchRateLimiter(long msgs, long bytes) { + return new ResourceGroupDispatchLimiter(msgs, bytes); + } + + static void updateReplicationDispatchRateLimiter( + ResourceGroupDispatchLimiter resourceGroupDispatchLimiter, + org.apache.pulsar.common.policies.data.ResourceGroup resourceGroup) { + long msgs = Optional.ofNullable(resourceGroup.getReplicationDispatchRateInMsgs()).orElse(-1L); + long bytes = Optional.ofNullable(resourceGroup.getReplicationDispatchRateInBytes()).orElse(-1L); + resourceGroupDispatchLimiter.update(msgs, bytes); + } + + static void updateReplicationDispatchRateLimiter(ResourceGroupDispatchLimiter resourceGroupDispatchLimiter, + BytesAndMessagesCount quota) { + resourceGroupDispatchLimiter.update(quota.messages, quota.bytes); + } + + static ResourceGroupDispatchLimiter newDispatchRateLimiter( + org.apache.pulsar.common.policies.data.ResourceGroup resourceGroup) { + long msgs = Optional.ofNullable(resourceGroup.getDispatchRateInMsgs()).orElse(-1); + long bytes = Optional.ofNullable(resourceGroup.getDispatchRateInBytes()).orElse(-1L); + return new ResourceGroupDispatchLimiter(msgs, bytes); + } + + static void updateDispatchRateLimiter(ResourceGroupDispatchLimiter resourceGroupDispatchLimiter, + org.apache.pulsar.common.policies.data.ResourceGroup resourceGroup) { + long msgs = Optional.ofNullable(resourceGroup.getDispatchRateInMsgs()).orElse(-1); + long bytes = Optional.ofNullable(resourceGroup.getDispatchRateInBytes()).orElse(-1L); + resourceGroupDispatchLimiter.update(msgs, bytes); + } + + static void updateDispatchRateLimiter(ResourceGroupDispatchLimiter resourceGroupDispatchLimiter, + BytesAndMessagesCount quota) { + resourceGroupDispatchLimiter.update(quota.messages, quota.bytes); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java index 379f9f870ea70..aa642eb79a4e6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java @@ -19,14 +19,26 @@ package org.apache.pulsar.broker.resourcegroup; import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.annotations.VisibleForTesting; import io.prometheus.client.Counter; +import io.prometheus.client.Gauge; import io.prometheus.client.Summary; +import io.prometheus.client.Summary.Child.Value; +import io.prometheus.client.Summary.Timer; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; import lombok.Getter; import lombok.val; import org.apache.pulsar.broker.PulsarService; @@ -38,6 +50,7 @@ import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; import org.slf4j.Logger; @@ -60,15 +73,16 @@ public class ResourceGroupService implements AutoCloseable{ public ResourceGroupService(PulsarService pulsar) { this.pulsar = pulsar; this.timeUnitScale = TimeUnit.SECONDS; - this.quotaCalculator = new ResourceQuotaCalculatorImpl(); + this.quotaCalculator = new ResourceQuotaCalculatorImpl(pulsar); this.resourceUsageTransportManagerMgr = pulsar.getResourceUsageTransportManager(); this.rgConfigListener = new ResourceGroupConfigListener(this, pulsar); this.initialize(); } // For testing only. + @VisibleForTesting public ResourceGroupService(PulsarService pulsar, TimeUnit timescale, - ResourceUsageTopicTransportManager transportMgr, + ResourceUsageTransportManager transportMgr, ResourceQuotaCalculator quotaCalc) { this.pulsar = pulsar; this.timeUnitScale = timescale; @@ -98,7 +112,7 @@ protected enum ResourceGroupUsageStatsType { * @throws if RG with that name already exists. */ public void resourceGroupCreate(String rgName, org.apache.pulsar.common.policies.data.ResourceGroup rgConfig) - throws PulsarAdminException { + throws PulsarAdminException { this.checkRGCreateParams(rgName, rgConfig); ResourceGroup rg = new ResourceGroup(this, rgName, rgConfig); resourceGroupsMap.put(rgName, rg); @@ -122,13 +136,7 @@ public void resourceGroupCreate(String rgName, * Get a copy of the RG with the given name. */ public ResourceGroup resourceGroupGet(String resourceGroupName) { - ResourceGroup retrievedRG = this.getResourceGroupInternal(resourceGroupName); - if (retrievedRG == null) { - return null; - } - - // Return a copy. - return new ResourceGroup(retrievedRG); + return this.getResourceGroupInternal(resourceGroupName); } /** @@ -137,7 +145,7 @@ public ResourceGroup resourceGroupGet(String resourceGroupName) { * @throws if RG with that name does not exist. */ public void resourceGroupUpdate(String rgName, org.apache.pulsar.common.policies.data.ResourceGroup rgConfig) - throws PulsarAdminException { + throws PulsarAdminException { if (rgConfig == null) { throw new IllegalArgumentException("ResourceGroupUpdate: Invalid null ResourceGroup config"); } @@ -154,6 +162,22 @@ public Set resourceGroupGetAll() { return resourceGroupsMap.keySet(); } + public void checkResourceGroupInUse(String name) throws PulsarAdminException { + ResourceGroup rg = this.getResourceGroupInternal(name); + if (rg == null) { + return; + } + long tenantRefCount = rg.getResourceGroupNumOfTenantRefs(); + long nsRefCount = rg.getResourceGroupNumOfNSRefs(); + long topicRefCount = rg.getResourceGroupNumOfTopicRefs(); + if ((tenantRefCount + nsRefCount + topicRefCount) > 0) { + String errMesg = "Resource group " + name + " still has " + tenantRefCount + " tenant refs"; + errMesg += " and " + nsRefCount + " namespace refs on it"; + errMesg += " and " + topicRefCount + " topic refs on it"; + throw new PulsarAdminException(errMesg); + } + } + /** * Delete RG. * @@ -164,16 +188,12 @@ public void resourceGroupDelete(String name) throws PulsarAdminException { if (rg == null) { throw new PulsarAdminException("Resource group does not exist: " + name); } - - long tenantRefCount = rg.getResourceGroupNumOfTenantRefs(); - long nsRefCount = rg.getResourceGroupNumOfNSRefs(); - if ((tenantRefCount + nsRefCount) > 0) { - String errMesg = "Resource group " + name + " still has " + tenantRefCount + " tenant refs"; - errMesg += " and " + nsRefCount + " namespace refs on it"; - throw new PulsarAdminException(errMesg); + checkResourceGroupInUse(name); + try { + rg.close(); + } catch (Exception e) { + log.warn("Failed to close resource group {}", rg.getID(), e); } - - rg.resourceGroupPublishLimiter = null; resourceGroupsMap.remove(name); } @@ -203,7 +223,7 @@ public void registerTenant(String resourceGroupName, String tenantName) throws P } ResourceGroupOpStatus status = rg.registerUsage(tenantName, ResourceGroupRefTypes.Tenants, true, - this.resourceUsageTransportManagerMgr); + this.resourceUsageTransportManagerMgr); if (status == ResourceGroupOpStatus.Exists) { String errMesg = "Tenant " + tenantName + " already references the resource group " + resourceGroupName; errMesg += "; this is unexpected"; @@ -229,7 +249,7 @@ public void unRegisterTenant(String resourceGroupName, String tenantName) throws ResourceGroup rg = checkResourceGroupExists(resourceGroupName); ResourceGroupOpStatus status = rg.registerUsage(tenantName, ResourceGroupRefTypes.Tenants, false, - this.resourceUsageTransportManagerMgr); + this.resourceUsageTransportManagerMgr); if (status == ResourceGroupOpStatus.DoesNotExist) { String errMesg = "Tenant " + tenantName + " does not yet reference resource group " + resourceGroupName; throw new PulsarAdminException(errMesg); @@ -247,7 +267,7 @@ public void unRegisterTenant(String resourceGroupName, String tenantName) throws * Registers a namespace as a user of a resource group. * * @param resourceGroupName - * @param fqNamespaceName (i.e., in "tenant/Namespace" format) + * @param fqNamespaceName (i.e., in "tenant/Namespace" format) * @throws if the RG does not exist, or if the NS already references the RG. */ public void registerNameSpace(String resourceGroupName, NamespaceName fqNamespaceName) throws PulsarAdminException { @@ -281,7 +301,7 @@ public void registerNameSpace(String resourceGroupName, NamespaceName fqNamespac * UnRegisters a namespace from a resource group. * * @param resourceGroupName - * @param fqNamespaceName i.e., in "tenant/Namespace" format) + * @param fqNamespaceName i.e., in "tenant/Namespace" format) * @throws if the RG does not exist, or if the NS does not references the RG yet. */ public void unRegisterNameSpace(String resourceGroupName, NamespaceName fqNamespaceName) @@ -296,6 +316,37 @@ public void unRegisterNameSpace(String resourceGroupName, NamespaceName fqNamesp throw new PulsarAdminException(errMesg); } + aggregateLock.lock(); + + Set invalidateAllKeyForProduce = new HashSet<>(); + topicProduceStats.asMap().forEach((key, value) -> { + TopicName topicName = TopicName.get(key); + if (topicName.getNamespaceObject().equals(fqNamespaceName)) { + invalidateAllKeyForProduce.add(key); + } + }); + topicProduceStats.invalidateAll(invalidateAllKeyForProduce); + + Set invalidateAllKeyForReplication = new HashSet<>(); + topicToReplicatorsMap.forEach((key, value) -> { + TopicName topicName = TopicName.get(key); + if (topicName.getNamespaceObject().equals(fqNamespaceName)) { + topicToReplicatorsMap.remove(key); + value.forEach(n -> invalidateAllKeyForReplication.add(getReplicatorKey(topicName.toString(), n))); + } + }); + replicationDispatchStats.invalidateAll(invalidateAllKeyForReplication); + + Set invalidateAllKeyForConsumer = new HashSet<>(); + topicConsumeStats.asMap().forEach((key, value) -> { + TopicName topicName = TopicName.get(key); + if (topicName.getNamespaceObject().equals(fqNamespaceName)) { + invalidateAllKeyForConsumer.add(key); + } + }); + topicConsumeStats.invalidateAll(invalidateAllKeyForConsumer); + + aggregateLock.unlock(); // Dissociate this NS-name from the RG. this.namespaceToRGsMap.remove(fqNamespaceName, rg); rgNamespaceUnRegisters.labels(resourceGroupName).inc(); @@ -304,6 +355,56 @@ public void unRegisterNameSpace(String resourceGroupName, NamespaceName fqNamesp maybeStopSchedulersIfIdle(); } + /** + * Registers a topic as a user of a resource group. + * + * @param resourceGroupName + * @param topicName complete topic name + */ + public void registerTopic(String resourceGroupName, TopicName topicName) { + ResourceGroup rg = resourceGroupsMap.get(resourceGroupName); + if (rg == null) { + throw new IllegalStateException("Resource group does not exist: " + resourceGroupName); + } + + ResourceGroupOpStatus status = rg.registerUsage(topicName.toString(), ResourceGroupRefTypes.Topics, + true, this.resourceUsageTransportManagerMgr); + if (status == ResourceGroupOpStatus.Exists) { + String msg = String.format("Topic %s already references the target resource group %s", + topicName, resourceGroupName); + throw new IllegalStateException(msg); + } + + // Associate this topic-name with the RG. + this.topicToRGsMap.put(topicName, rg); + rgTopicRegisters.labels(resourceGroupName).inc(); + } + + /** + * UnRegisters a topic from a resource group. + * + * @param topicName complete topic name + */ + public void unRegisterTopic(TopicName topicName) { + aggregateLock.lock(); + String topicNameString = topicName.toString(); + ResourceGroup remove = topicToRGsMap.remove(topicName); + if (remove != null) { + remove.registerUsage(topicNameString, ResourceGroupRefTypes.Topics, + false, this.resourceUsageTransportManagerMgr); + rgTopicUnRegisters.labels(remove.resourceGroupName).inc(); + } + topicProduceStats.invalidate(topicNameString); + topicConsumeStats.invalidate(topicNameString); + Set replicators = topicToReplicatorsMap.remove(topicNameString); + if (replicators != null) { + List keys = replicators.stream().map(n -> getReplicatorKey(topicNameString, n)) + .collect(Collectors.toList()); + replicationDispatchStats.invalidateAll(keys); + } + aggregateLock.unlock(); + } + /** * Return the resource group associated with a namespace. * @@ -314,6 +415,11 @@ public ResourceGroup getNamespaceResourceGroup(NamespaceName namespaceName) { return this.namespaceToRGsMap.get(namespaceName); } + @VisibleForTesting + public ResourceGroup getTopicResourceGroup(TopicName topicName) { + return this.topicToRGsMap.get(topicName); + } + @Override public void close() throws Exception { if (aggregateLocalUsagePeriodicTask != null) { @@ -331,12 +437,26 @@ public void close() throws Exception { resourceGroupsMap.clear(); tenantToRGsMap.clear(); namespaceToRGsMap.clear(); - topicProduceStats.clear(); - topicConsumeStats.clear(); + topicProduceStats.invalidateAll(); + topicConsumeStats.invalidateAll(); + replicationDispatchStats.invalidateAll(); + } + + private void incrementUsage(ResourceGroup resourceGroup, + ResourceGroupMonitoringClass monClass, BytesAndMessagesCount incStats, + String remoteCluster) + throws PulsarAdminException { + resourceGroup.incrementLocalUsageStats(monClass, incStats, remoteCluster); + rgLocalUsageBytes.labels(resourceGroup.resourceGroupName, monClass.name(), + pulsar.getConfiguration().getClusterName(), remoteCluster != null ? remoteCluster : "") + .inc(incStats.bytes); + rgLocalUsageMessages.labels(resourceGroup.resourceGroupName, monClass.name(), + pulsar.getConfiguration().getClusterName(), remoteCluster != null ? remoteCluster : "") + .inc(incStats.messages); } /** - * Increments usage stats for the resource groups associated with the given namespace and tenant. + * Increments usage stats for the resource groups associated with the given namespace, tenant, and topic. * Expected to be called when a message is produced or consumed on a topic, or when we calculate * usage periodically in the background by going through broker-service stats. [Not yet decided * which model we will follow.] Broker-service stats will be cumulative, while calls from the @@ -344,22 +464,25 @@ public void close() throws Exception { * * If the tenant and NS are associated with different RGs, the statistics on both RGs are updated. * If the tenant and NS are associated with the same RG, the stats on the RG are updated only once + * If the tenant, NS and topic are associated with the same RG, the stats on the RG are updated only once * (to avoid a direct double-counting). * ToDo: will this distinction result in "expected semantics", or shock from users? * For now, the only caller is internal to this class. * * @param tenantName - * @param nsName + * @param nsName Complete namespace name + * @param topicName Complete topic name * @param monClass * @param incStats * @returns true if the stats were updated; false if nothing was updated. */ - protected boolean incrementUsage(String tenantName, String nsName, - ResourceGroupMonitoringClass monClass, - BytesAndMessagesCount incStats) throws PulsarAdminException { - final ResourceGroup nsRG = this.namespaceToRGsMap.get(NamespaceName.get(tenantName, nsName)); + protected boolean incrementUsage(String tenantName, String nsName, String topicName, + ResourceGroupMonitoringClass monClass, + BytesAndMessagesCount incStats, String remoteCluster) throws PulsarAdminException { + final ResourceGroup nsRG = this.namespaceToRGsMap.get(NamespaceName.get(nsName)); final ResourceGroup tenantRG = this.tenantToRGsMap.get(tenantName); - if (tenantRG == null && nsRG == null) { + final ResourceGroup topicRG = this.topicToRGsMap.get(TopicName.get(topicName)); + if (tenantRG == null && nsRG == null && topicRG == null) { return false; } @@ -370,51 +493,60 @@ protected boolean incrementUsage(String tenantName, String nsName, throw new PulsarAdminException(errMesg); } - if (nsRG == tenantRG) { + if (tenantRG == nsRG && nsRG == topicRG) { // Update only once in this case. - // Note that we will update both tenant and namespace RGs in other cases. - nsRG.incrementLocalUsageStats(monClass, incStats); - rgLocalUsageMessages.labels(nsRG.resourceGroupName, monClass.name()).inc(incStats.messages); - rgLocalUsageBytes.labels(nsRG.resourceGroupName, monClass.name()).inc(incStats.bytes); + // Note that we will update both tenant, namespace and topic RGs in other cases. + incrementUsage(tenantRG, monClass, incStats, remoteCluster); + return true; + } + + if (tenantRG != null && tenantRG == nsRG) { + // Tenant and Namespace GRs are same. + incrementUsage(tenantRG, monClass, incStats, remoteCluster); + if (topicRG == null) { + return true; + } + } + + if (nsRG != null && nsRG == topicRG) { + // Namespace and Topic GRs are same. + incrementUsage(nsRG, monClass, incStats, remoteCluster); return true; } if (tenantRG != null) { - tenantRG.incrementLocalUsageStats(monClass, incStats); - rgLocalUsageMessages.labels(tenantRG.resourceGroupName, monClass.name()).inc(incStats.messages); - rgLocalUsageBytes.labels(tenantRG.resourceGroupName, monClass.name()).inc(incStats.bytes); + // Tenant GR is different from other resource GR. + incrementUsage(tenantRG, monClass, incStats, remoteCluster); } + if (nsRG != null) { - nsRG.incrementLocalUsageStats(monClass, incStats); - rgLocalUsageMessages.labels(nsRG.resourceGroupName, monClass.name()).inc(incStats.messages); - rgLocalUsageBytes.labels(nsRG.resourceGroupName, monClass.name()).inc(incStats.bytes); + // Namespace GR is different from other resource GR. + incrementUsage(nsRG, monClass, incStats, remoteCluster); + } + + if (topicRG != null) { + // Topic GR is different from other resource GR. + incrementUsage(topicRG, monClass, incStats, remoteCluster); } return true; } // Visibility for testing. - protected BytesAndMessagesCount getRGUsage(String rgName, ResourceGroupMonitoringClass monClass, - ResourceGroupUsageStatsType statsType) throws PulsarAdminException { + protected Map getRGUsage(String rgName, ResourceGroupMonitoringClass monClass, + ResourceGroupUsageStatsType statsType) + throws PulsarAdminException { final ResourceGroup rg = this.getResourceGroupInternal(rgName); if (rg != null) { switch (statsType) { + case Cumulative: + return rg.getLocalUsageStatsCumulative(monClass); default: String errStr = "Unsupported statsType: " + statsType; throw new PulsarAdminException(errStr); - case Cumulative: - return rg.getLocalUsageStatsCumulative(monClass); - case LocalSinceLastReported: - return rg.getLocalUsageStats(monClass); - case ReportFromTransportMgr: - return rg.getLocalUsageStatsFromBrokerReports(monClass); } } - - BytesAndMessagesCount retCount = new BytesAndMessagesCount(); - retCount.bytes = -1; - retCount.messages = -1; - return retCount; + return Collections.emptyMap(); } /** @@ -436,11 +568,17 @@ private ResourceGroup checkResourceGroupExists(String rgName) throws PulsarAdmin return rg; } + private String getReplicatorKey(String topic, String replicationRemoteCluster) { + return topic + replicationRemoteCluster; + } + // Find the difference between the last time stats were updated for this topic, and the current // time. If the difference is positive, update the stats. - private void updateStatsWithDiff(String topicName, String tenantString, String nsString, - long accByteCount, long accMesgCount, ResourceGroupMonitoringClass monClass) { - ConcurrentHashMap hm; + @VisibleForTesting + protected void updateStatsWithDiff(String topicName, String replicationRemoteCluster, String tenantString, + String nsString, long accByteCount, long accMsgCount, + ResourceGroupMonitoringClass monClass) { + Cache hm; switch (monClass) { default: log.error("updateStatsWithDiff: Unknown monitoring class={}; ignoring", monClass); @@ -453,6 +591,10 @@ private void updateStatsWithDiff(String topicName, String tenantString, String n case Dispatch: hm = this.topicConsumeStats; break; + + case ReplicationDispatch: + hm = this.replicationDispatchStats; + break; } BytesAndMessagesCount bmDiff = new BytesAndMessagesCount(); @@ -460,9 +602,22 @@ private void updateStatsWithDiff(String topicName, String tenantString, String n BytesAndMessagesCount bmNewCount = new BytesAndMessagesCount(); bmNewCount.bytes = accByteCount; - bmNewCount.messages = accMesgCount; - - bmOldCount = hm.get(topicName); + bmNewCount.messages = accMsgCount; + + String key; + if (monClass == ResourceGroupMonitoringClass.ReplicationDispatch) { + key = getReplicatorKey(topicName, replicationRemoteCluster); + topicToReplicatorsMap.compute(topicName, (n, value) -> { + if (value == null) { + value = new CopyOnWriteArraySet<>(); + } + value.add(replicationRemoteCluster); + return value; + }); + } else { + key = topicName; + } + bmOldCount = hm.getIfPresent(key); if (bmOldCount == null) { bmDiff.bytes = bmNewCount.bytes; bmDiff.messages = bmNewCount.messages; @@ -476,14 +631,15 @@ private void updateStatsWithDiff(String topicName, String tenantString, String n } try { - boolean statsUpdated = this.incrementUsage(tenantString, nsString, monClass, bmDiff); + boolean statsUpdated = + this.incrementUsage(tenantString, nsString, topicName, monClass, bmDiff, replicationRemoteCluster); if (log.isDebugEnabled()) { log.debug("updateStatsWithDiff for topic={}: monclass={} statsUpdated={} for tenant={}, namespace={}; " + "by {} bytes, {} mesgs", topicName, monClass, statsUpdated, tenantString, nsString, bmDiff.bytes, bmDiff.messages); } - hm.put(topicName, bmNewCount); + hm.put(key, bmNewCount); } catch (Throwable t) { log.error("updateStatsWithDiff: got ex={} while aggregating for {} side", t.getMessage(), monClass); @@ -491,7 +647,7 @@ private void updateStatsWithDiff(String topicName, String tenantString, String n } // Visibility for testing. - protected BytesAndMessagesCount getPublishRateLimiters (String rgName) throws PulsarAdminException { + protected BytesAndMessagesCount getPublishRateLimiters(String rgName) throws PulsarAdminException { ResourceGroup rg = this.getResourceGroupInternal(rgName); if (rg == null) { throw new PulsarAdminException("Resource group does not exist: " + rgName); @@ -501,23 +657,51 @@ protected BytesAndMessagesCount getPublishRateLimiters (String rgName) throws Pu } // Visibility for testing. - protected static long getRgQuotaByteCount (String rgName, String monClassName) { - return (long) rgCalculatedQuotaBytes.labels(rgName, monClassName).get(); + @VisibleForTesting + protected static long getRgQuotaByteCount(String rgName, String monClassName, String localCluster, + String remoteCluster) { + return (long) rgCalculatedQuotaBytes.labels(rgName, monClassName, localCluster, + remoteCluster != null ? remoteCluster : "").get(); + } + + // Visibility for testing. + @VisibleForTesting + protected static long getRgQuotaByte(String rgName, String monClassName, String localCluster, + String remoteCluster) { + return (long) rgCalculatedQuotaBytesGauge.labels(rgName, monClassName, localCluster, + remoteCluster != null ? remoteCluster : "").get(); + } + + // Visibility for testing. + @VisibleForTesting + protected static long getRgQuotaMessageCount(String rgName, String monClassName, String localCluster, + String remoteCluster) { + return (long) rgCalculatedQuotaMessages.labels(rgName, monClassName, localCluster, + remoteCluster != null ? remoteCluster : "").get(); } // Visibility for testing. - protected static long getRgQuotaMessageCount (String rgName, String monClassName) { - return (long) rgCalculatedQuotaMessages.labels(rgName, monClassName).get(); + @VisibleForTesting + protected static double getRgQuotaMessage(String rgName, String monClassName, String localCluster, + String remoteCluster) { + return rgCalculatedQuotaMessagesGauge.labels(rgName, monClassName, localCluster, + remoteCluster != null ? remoteCluster : "").get(); } // Visibility for testing. - protected static long getRgLocalUsageByteCount (String rgName, String monClassName) { - return (long) rgLocalUsageBytes.labels(rgName, monClassName).get(); + @VisibleForTesting + protected static long getRgLocalUsageByteCount(String rgName, String monClassName, String localCluster, + String remoteCluster) { + return (long) rgLocalUsageBytes.labels(rgName, monClassName, localCluster, remoteCluster != null ? remoteCluster : "") + .get(); } // Visibility for testing. - protected static long getRgLocalUsageMessageCount (String rgName, String monClassName) { - return (long) rgLocalUsageMessages.labels(rgName, monClassName).get(); + @VisibleForTesting + protected static long getRgLocalUsageMessageCount(String rgName, String monClassName, String localCluster, + String remoteCluster) { + return (long) rgLocalUsageMessages.labels(rgName, monClassName, localCluster, + remoteCluster != null ? remoteCluster : "").get(); } // Visibility for testing. @@ -526,32 +710,32 @@ protected static long getRgUpdatesCount (String rgName) { } // Visibility for testing. - protected static long getRgTenantRegistersCount (String rgName) { + protected static long getRgTenantRegistersCount(String rgName) { return (long) rgTenantRegisters.labels(rgName).get(); } // Visibility for testing. - protected static long getRgTenantUnRegistersCount (String rgName) { + protected static long getRgTenantUnRegistersCount(String rgName) { return (long) rgTenantUnRegisters.labels(rgName).get(); } // Visibility for testing. - protected static long getRgNamespaceRegistersCount (String rgName) { + protected static long getRgNamespaceRegistersCount(String rgName) { return (long) rgNamespaceRegisters.labels(rgName).get(); } // Visibility for testing. - protected static long getRgNamespaceUnRegistersCount (String rgName) { + protected static long getRgNamespaceUnRegistersCount(String rgName) { return (long) rgNamespaceUnRegisters.labels(rgName).get(); } // Visibility for testing. - protected static Summary.Child.Value getRgUsageAggregationLatency() { + protected static Value getRgUsageAggregationLatency() { return rgUsageAggregationLatency.get(); } // Visibility for testing. - protected static Summary.Child.Value getRgQuotaCalculationTime() { + protected static Value getRgQuotaCalculationTime() { return rgQuotaCalculationLatency.get(); } @@ -561,34 +745,44 @@ protected void aggregateResourceGroupLocalUsages() { if (!shouldRunPeriodicTasks()) { return; } - final Summary.Timer aggrUsageTimer = rgUsageAggregationLatency.startTimer(); + final Timer aggrUsageTimer = rgUsageAggregationLatency.startTimer(); BrokerService bs = this.pulsar.getBrokerService(); Map topicStatsMap = bs.getTopicStats(); - for (Map.Entry entry : topicStatsMap.entrySet()) { + aggregateLock.lock(); + for (Entry entry : topicStatsMap.entrySet()) { final String topicName = entry.getKey(); final TopicStats topicStats = entry.getValue(); final TopicName topic = TopicName.get(topicName); final String tenantString = topic.getTenant(); - final String nsString = topic.getNamespacePortion(); + final String nsString = topic.getNamespace(); final NamespaceName fqNamespace = topic.getNamespaceObject(); // Can't use containsKey here, as that checks for exact equality // (we need a check for string-comparison). val tenantRG = this.tenantToRGsMap.get(tenantString); val namespaceRG = this.namespaceToRGsMap.get(fqNamespace); - if (tenantRG == null && namespaceRG == null) { + val topicRG = this.topicToRGsMap.get(topic); + if (tenantRG == null && namespaceRG == null && topicRG == null) { // This topic's NS/tenant are not registered to any RG. continue; } - this.updateStatsWithDiff(topicName, tenantString, nsString, + topicStats.getReplication().forEach((remoteCluster, stats) -> { + this.updateStatsWithDiff(topicName, remoteCluster, tenantString, nsString, + stats.getBytesOutCount(), + stats.getMsgOutCount(), + ResourceGroupMonitoringClass.ReplicationDispatch + ); + }); + this.updateStatsWithDiff(topicName, null, tenantString, nsString, topicStats.getBytesInCounter(), topicStats.getMsgInCounter(), ResourceGroupMonitoringClass.Publish); - this.updateStatsWithDiff(topicName, tenantString, nsString, + this.updateStatsWithDiff(topicName, null, tenantString, nsString, topicStats.getBytesOutCounter(), topicStats.getMsgOutCounter(), ResourceGroupMonitoringClass.Dispatch); } + aggregateLock.unlock(); double diffTimeSeconds = aggrUsageTimer.observeDuration(); if (log.isDebugEnabled()) { log.debug("aggregateResourceGroupLocalUsages took {} milliseconds", diffTimeSeconds * 1000); @@ -627,55 +821,11 @@ protected void calculateQuotaForAllResourceGroups() { return; } // Calculate the quota for the next window for this RG, based on the observed usage. - final Summary.Timer quotaCalcTimer = rgQuotaCalculationLatency.startTimer(); - BytesAndMessagesCount updatedQuota = new BytesAndMessagesCount(); + final Timer quotaCalcTimer = rgQuotaCalculationLatency.startTimer(); this.resourceGroupsMap.forEach((rgName, resourceGroup) -> { - BytesAndMessagesCount globalUsageStats; - BytesAndMessagesCount localUsageStats; - BytesAndMessagesCount confCounts; for (ResourceGroupMonitoringClass monClass : ResourceGroupMonitoringClass.values()) { try { - globalUsageStats = resourceGroup.getGlobalUsageStats(monClass); - localUsageStats = resourceGroup.getLocalUsageStatsFromBrokerReports(monClass); - confCounts = resourceGroup.getConfLimits(monClass); - - long[] globUsageBytesArray = new long[] { globalUsageStats.bytes }; - updatedQuota.bytes = this.quotaCalculator.computeLocalQuota( - confCounts.bytes, - localUsageStats.bytes, - globUsageBytesArray); - - long[] globUsageMessagesArray = new long[] {globalUsageStats.messages }; - updatedQuota.messages = this.quotaCalculator.computeLocalQuota( - confCounts.messages, - localUsageStats.messages, - globUsageMessagesArray); - - BytesAndMessagesCount oldBMCount = resourceGroup.updateLocalQuota(monClass, updatedQuota); - // Guard against unconfigured quota settings, for which computeLocalQuota will return negative. - if (updatedQuota.messages >= 0) { - rgCalculatedQuotaMessages.labels(rgName, monClass.name()).inc(updatedQuota.messages); - } - if (updatedQuota.bytes >= 0) { - rgCalculatedQuotaBytes.labels(rgName, monClass.name()).inc(updatedQuota.bytes); - } - if (oldBMCount != null) { - long messagesIncrement = updatedQuota.messages - oldBMCount.messages; - long bytesIncrement = updatedQuota.bytes - oldBMCount.bytes; - if (log.isDebugEnabled()) { - log.debug("calculateQuota for RG={} [class {}]: " - + "updatedlocalBytes={}, updatedlocalMesgs={}; " - + "old bytes={}, old mesgs={}; incremented bytes by {}, messages by {}", - rgName, monClass, updatedQuota.bytes, updatedQuota.messages, - oldBMCount.bytes, oldBMCount.messages, - bytesIncrement, messagesIncrement); - } - } else { - if (log.isDebugEnabled()) { - log.debug("calculateQuota for RG={} [class {}]: got back null from updateLocalQuota", - rgName, monClass); - } - } + calculateQuotaByMonClass(rgName, resourceGroup, monClass); } catch (Throwable t) { log.error("Got exception={} while calculating new quota for monitoring-class={} of RG={}", t.getMessage(), monClass, rgName); @@ -700,23 +850,22 @@ protected void calculateQuotaForAllResourceGroups() { } else { boolean cancelStatus = this.calculateQuotaPeriodicTask.cancel(true); log.info("calculateQuotaForAllResourceGroups: Got status={} in cancel of periodic " - + " when publish period changed from {} to {} {}", + + " when publish period changed from {} to {} {}", cancelStatus, this.resourceUsagePublishPeriodInSeconds, newPeriodInSeconds, timeUnitScale); } this.calculateQuotaPeriodicTask = pulsar.getExecutor().scheduleAtFixedRate( - catchingAndLoggingThrowables(this::calculateQuotaForAllResourceGroups), - newPeriodInSeconds, - newPeriodInSeconds, - timeUnitScale); + catchingAndLoggingThrowables(this::calculateQuotaForAllResourceGroups), + newPeriodInSeconds, + newPeriodInSeconds, + timeUnitScale); this.resourceUsagePublishPeriodInSeconds = newPeriodInSeconds; - maxIntervalForSuppressingReportsMSecs = - TimeUnit.SECONDS.toMillis(this.resourceUsagePublishPeriodInSeconds) * MaxUsageReportSuppressRounds; } } // Returns true if at least one tenant or namespace is registered to resource group. private boolean hasActiveResourceGroups() { - return !tenantToRGsMap.isEmpty() || !namespaceToRGsMap.isEmpty(); + return !tenantToRGsMap.isEmpty() || !namespaceToRGsMap.isEmpty() || !topicToRGsMap.isEmpty() + || !topicToReplicatorsMap.isEmpty(); } /** @@ -746,44 +895,132 @@ private void maybeStartSchedulers() { this.calculateQuotaPeriodicTask = pulsar.getExecutor().scheduleAtFixedRate( catchingAndLoggingThrowables(this::calculateQuotaForAllResourceGroups), periodInSecs, periodInSecs, timeUnitScale); - maxIntervalForSuppressingReportsMSecs = - TimeUnit.SECONDS.toMillis(this.resourceUsagePublishPeriodInSeconds) * MaxUsageReportSuppressRounds; if (log.isInfoEnabled()) { log.info("Started ResourceGroupService periodic tasks with period={} {}", periodInSecs, timeUnitScale); } } } - // Stop schedulers when no tenant or namespace registrations remain. - private void maybeStopSchedulersIfIdle() { - if (hasActiveResourceGroups()) { - return; + + @VisibleForTesting + protected void calculateQuotaByMonClass(String rgName, ResourceGroup resourceGroup, + ResourceGroupMonitoringClass monClass) throws PulsarAdminException { + Map globalUsageStats = + resourceGroup.getGlobalUsageStats(monClass); + Map localUsageStats = + resourceGroup.getLocalUsageStatsFromBrokerReports(monClass); + Map confLimits = + resourceGroup.getConfLimits(monClass); + + org.apache.pulsar.common.policies.data.ResourceGroup rgConfig = resourceGroup.getRgConfig(); + Set specificClusters; + if (monClass.equals(ResourceGroupMonitoringClass.ReplicationDispatch)) { + Map replicatorDispatchRateMap = rgConfig.getReplicatorDispatchRate(); + specificClusters = + !replicatorDispatchRateMap.isEmpty() ? replicatorDispatchRateMap.keySet() : Collections.emptySet(); + } else { + specificClusters = Collections.emptySet(); } - if (schedulersRunning.compareAndSet(true, false)) { - if (aggregateLocalUsagePeriodicTask != null) { - aggregateLocalUsagePeriodicTask.cancel(true); - aggregateLocalUsagePeriodicTask = null; + + // Accumulate local and global usage for clusters that do NOT have specific per-cluster limits. + long totalGlobalBytesLocal = 0; + long totalGlobalMsgsLocal = 0; + long totalGlobalBytes = 0; + long totalGlobalMsgs = 0; + Long globalByteLimit = null; + Long globalMsgLimit = null; + for (Entry entry : localUsageStats.entrySet()) { + String remoteCluster = entry.getKey(); + if (!specificClusters.contains(resourceGroup.getReplicatorDispatchRateLimiterKey(remoteCluster))) { + BytesAndMessagesCount local = entry.getValue(); + totalGlobalBytesLocal += local.bytes; + totalGlobalMsgsLocal += local.messages; + BytesAndMessagesCount global = globalUsageStats.get(remoteCluster); + if (global != null) { + totalGlobalBytes += global.bytes; + totalGlobalMsgs += global.messages; + } + // Get configured limits + if (globalByteLimit == null && globalMsgLimit == null) { + BytesAndMessagesCount conf = confLimits.get(remoteCluster); + if (conf != null) { + globalByteLimit = conf.bytes; + globalMsgLimit = conf.messages; + } + } } - if (calculateQuotaPeriodicTask != null) { - calculateQuotaPeriodicTask.cancel(true); - calculateQuotaPeriodicTask = null; + } + + // Compute global quota + BytesAndMessagesCount globalQuota = new BytesAndMessagesCount(); + globalQuota.bytes = this.quotaCalculator.computeLocalQuota(globalByteLimit == null ? 0 : globalByteLimit, + totalGlobalBytesLocal, + new long[]{totalGlobalBytes}); + globalQuota.messages = this.quotaCalculator.computeLocalQuota(globalMsgLimit == null ? 0 : globalMsgLimit, + totalGlobalMsgsLocal, + new long[]{totalGlobalMsgs}); + for (Entry localUsageEntry : localUsageStats.entrySet()) { + String remoteCluster = localUsageEntry.getKey(); + BytesAndMessagesCount quota = null; + if (monClass.equals(ResourceGroupMonitoringClass.ReplicationDispatch)) { + if (!rgConfig.getReplicatorDispatchRate().isEmpty()) { + BytesAndMessagesCount localUsage = localUsageEntry.getValue(); + BytesAndMessagesCount globalUsage = + globalUsageStats.getOrDefault(remoteCluster, new BytesAndMessagesCount()); + if (specificClusters.contains(resourceGroup.getReplicatorDispatchRateLimiterKey(remoteCluster))) { + // Use specific config + BytesAndMessagesCount conf = + confLimits.getOrDefault(remoteCluster, new BytesAndMessagesCount()); + quota = new BytesAndMessagesCount(); + quota.bytes = this.quotaCalculator.computeLocalQuota(conf.bytes, localUsage.bytes, + new long[]{globalUsage.bytes}); + quota.messages = this.quotaCalculator.computeLocalQuota(conf.messages, localUsage.messages, + new long[]{globalUsage.messages}); + resourceGroup.updateLocalQuota(monClass, quota, remoteCluster); + } + } } - if (log.isInfoEnabled()) { - log.info("Stopped ResourceGroupService periodic tasks because no registrations remain"); + if (quota == null) { + resourceGroup.updateLocalQuota(monClass, globalQuota, null); // null means default/global } + String localCluster = pulsar.getConfiguration().getClusterName(); + incRgCalculatedQuota(rgName, monClass, quota == null ? globalQuota : quota, + resourceUsagePublishPeriodInSeconds, localCluster, + remoteCluster); } } private void initialize() { - // Store the configured interval. Do not start periodic tasks unconditionally here. - // Schedulers are started by maybeStartSchedulers() when the first tenant/namespace is registered. - final long periodInSecs = pulsar.getConfiguration().getResourceUsageTransportPublishIntervalInSecs(); - this.aggregateLocalUsagePeriodInSeconds = this.resourceUsagePublishPeriodInSeconds = periodInSecs; - // if any tenant/namespace registrations already exist, maybeStartSchedulers() will start the schedulers now. + long resourceUsagePublishPeriodInMS = TimeUnit.SECONDS.toMillis(this.resourceUsagePublishPeriodInSeconds); + long statsCacheInMS = resourceUsagePublishPeriodInMS * 2; + topicProduceStats = newStatsCache(statsCacheInMS); + topicConsumeStats = newStatsCache(statsCacheInMS); + replicationDispatchStats = newStatsCache(statsCacheInMS); + maybeStartSchedulers(); } + // Stop schedulers when no tenant or namespace registrations remain. + private void maybeStopSchedulersIfIdle () { + if (hasActiveResourceGroups()) { + return; + } + if (schedulersRunning.compareAndSet(true, false)) { + if (aggregateLocalUsagePeriodicTask != null) { + aggregateLocalUsagePeriodicTask.cancel(true); + aggregateLocalUsagePeriodicTask = null; + } + if (calculateQuotaPeriodicTask != null) { + calculateQuotaPeriodicTask.cancel(true); + calculateQuotaPeriodicTask = null; + } + if (log.isInfoEnabled()) { + log.info("Stopped ResourceGroupService periodic tasks because no registrations remain"); + } + } + } + private void checkRGCreateParams(String rgName, org.apache.pulsar.common.policies.data.ResourceGroup rgConfig) - throws PulsarAdminException { + throws PulsarAdminException { if (rgConfig == null) { throw new IllegalArgumentException("ResourceGroupCreate: Invalid null ResourceGroup config"); } @@ -798,6 +1035,33 @@ private void checkRGCreateParams(String rgName, org.apache.pulsar.common.policie } } + static void incRgCalculatedQuota(String rgName, ResourceGroupMonitoringClass monClass, + BytesAndMessagesCount updatedQuota, long resourceUsagePublishPeriodInSeconds, + String localCluster, String remoteCluster) { + // Guard against unconfigured quota settings, for which computeLocalQuota will return negative. + if (updatedQuota.messages >= 0) { + rgCalculatedQuotaMessages.labels(rgName, monClass.name(), localCluster, + remoteCluster != null ? remoteCluster : "") + .inc(updatedQuota.messages * resourceUsagePublishPeriodInSeconds); + rgCalculatedQuotaMessagesGauge.labels(rgName, monClass.name(), localCluster, + remoteCluster != null ? remoteCluster : "").set(updatedQuota.messages); + } + if (updatedQuota.bytes >= 0) { + rgCalculatedQuotaBytes.labels(rgName, monClass.name(), localCluster, + remoteCluster != null ? remoteCluster : "") + .inc(updatedQuota.bytes * resourceUsagePublishPeriodInSeconds); + rgCalculatedQuotaBytesGauge.labels(rgName, monClass.name(), localCluster, + remoteCluster != null ? remoteCluster : "").set(updatedQuota.bytes); + } + } + + @VisibleForTesting + protected Cache newStatsCache(long durationMS) { + return Caffeine.newBuilder() + .expireAfterAccess(durationMS, TimeUnit.MILLISECONDS) + .build(); + } + private static final Logger log = LoggerFactory.getLogger(ResourceGroupService.class); @Getter @@ -817,11 +1081,14 @@ private void checkRGCreateParams(String rgName, org.apache.pulsar.common.policie // Given a qualified NS-name (i.e., in "tenant/namespace" format), record its associated resource-group private ConcurrentHashMap namespaceToRGsMap = new ConcurrentHashMap<>(); + private ConcurrentHashMap topicToRGsMap = new ConcurrentHashMap<>(); + private ConcurrentHashMap> topicToReplicatorsMap = new ConcurrentHashMap<>(); + private ReentrantLock aggregateLock = new ReentrantLock(); // Maps to maintain the usage per topic, in produce/consume directions. - private ConcurrentHashMap topicProduceStats = new ConcurrentHashMap<>(); - private ConcurrentHashMap topicConsumeStats = new ConcurrentHashMap<>(); - + private Cache topicProduceStats; + private Cache topicConsumeStats; + private Cache replicationDispatchStats; // The task that periodically re-calculates the quota budget for local usage. private ScheduledFuture aggregateLocalUsagePeriodicTask; @@ -837,36 +1104,34 @@ private void checkRGCreateParams(String rgName, org.apache.pulsar.common.policie private final java.util.concurrent.atomic.AtomicBoolean schedulersRunning = new java.util.concurrent.atomic.AtomicBoolean(false); - // The maximum number of successive rounds that we can suppress reporting local usage, because there was no - // substantial change from the prior round. This is to ensure the reporting does not become too chatty. - // Set this value to one more than the cadence of sending reports; e.g., if you want to send every 3rd report, - // set the value to 4. - // Setting this to 0 will make us report in every round. - // Don't set to negative values; behavior will be "undefined". - protected static final int MaxUsageReportSuppressRounds = 5; - // Convenient shorthand, for MaxUsageReportSuppressRounds converted to a time interval in milliseconds. protected static long maxIntervalForSuppressingReportsMSecs; - // The percentage difference that is considered "within limits" to suppress usage reporting. - // Setting this to 0 will also make us report in every round. - // Don't set it to negative values; behavior will be "undefined". - protected static final float UsageReportSuppressionTolerancePercentage = 5; - // Labels for the various counters used here. private static final String[] resourceGroupLabel = {"ResourceGroup"}; - private static final String[] resourceGroupMonitoringclassLabels = {"ResourceGroup", "MonitoringClass"}; + private static final String[] resourceGroupMonitoringclassLabels = + {"ResourceGroup", "MonitoringClass", "LocalCluster", "RemoteCluster"}; private static final Counter rgCalculatedQuotaBytes = Counter.build() .name("pulsar_resource_group_calculated_bytes_quota") .help("Bytes quota calculated for resource group") .labelNames(resourceGroupMonitoringclassLabels) .register(); + private static final Gauge rgCalculatedQuotaBytesGauge = Gauge.build() + .name("pulsar_resource_group_calculated_bytes_quota_gauge") + .help("Bytes quota calculated for resource group") + .labelNames(resourceGroupMonitoringclassLabels) + .register(); private static final Counter rgCalculatedQuotaMessages = Counter.build() .name("pulsar_resource_group_calculated_messages_quota") .help("Messages quota calculated for resource group") .labelNames(resourceGroupMonitoringclassLabels) .register(); + private static final Gauge rgCalculatedQuotaMessagesGauge = Gauge.build() + .name("pulsar_resource_group_calculated_messages_quota_gauge") + .help("Messages quota calculated for resource group") + .labelNames(resourceGroupMonitoringclassLabels) + .register(); private static final Counter rgLocalUsageBytes = Counter.build() .name("pulsar_resource_group_bytes_used") @@ -907,6 +1172,17 @@ private void checkRGCreateParams(String rgName, org.apache.pulsar.common.policie .labelNames(resourceGroupLabel) .register(); + private static final Counter rgTopicRegisters = Counter.build() + .name("pulsar_resource_group_topic_registers") + .help("Number of registrations of topics") + .labelNames(resourceGroupLabel) + .register(); + private static final Counter rgTopicUnRegisters = Counter.build() + .name("pulsar_resource_group_topic_unregisters") + .help("Number of un-registrations of topics") + .labelNames(resourceGroupLabel) + .register(); + private static final Summary rgUsageAggregationLatency = Summary.build() .quantile(0.5, 0.05) .quantile(0.9, 0.01) @@ -922,15 +1198,25 @@ private void checkRGCreateParams(String rgName, org.apache.pulsar.common.policie .register(); @VisibleForTesting - ConcurrentHashMap getTopicConsumeStats() { + Cache getTopicConsumeStats() { return this.topicConsumeStats; } @VisibleForTesting - ConcurrentHashMap getTopicProduceStats() { + Cache getTopicProduceStats() { return this.topicProduceStats; } + @VisibleForTesting + Cache getReplicationDispatchStats() { + return this.replicationDispatchStats; + } + + @VisibleForTesting + ConcurrentHashMap> getTopicToReplicatorsMap() { + return this.topicToReplicatorsMap; + } + @VisibleForTesting ScheduledFuture getAggregateLocalUsagePeriodicTask() { return this.aggregateLocalUsagePeriodicTask; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImpl.java index b3000c9d77fe5..65433a7b79390 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImpl.java @@ -18,14 +18,22 @@ */ package org.apache.pulsar.broker.resourcegroup; -import static java.lang.Float.max; import static java.lang.Math.abs; +import java.util.concurrent.TimeUnit; import lombok.val; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.client.admin.PulsarAdminException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class ResourceQuotaCalculatorImpl implements ResourceQuotaCalculator { + private final PulsarService pulsarService; + + public ResourceQuotaCalculatorImpl(PulsarService pulsarService) { + this.pulsarService = pulsarService; + } + @Override public long computeLocalQuota(long confUsage, long myUsage, long[] allUsages) throws PulsarAdminException { // ToDo: work out the initial conditions: we may allow a small number of "first few iterations" to go @@ -53,13 +61,14 @@ public long computeLocalQuota(long confUsage, long myUsage, long[] allUsages) th throw new PulsarAdminException(errMesg); } - // If the total usage is zero (which may happen during initial transients), just return the configured value. + // If the total/myUsage usage is zero (which may happen during initial transients, or when there is no + // traffic in the current broker), just return the configured value. // The caller is expected to check the value returned, or not call here with a zero global usage. // [This avoids a division by zero when calculating the local share.] - if (totalUsage == 0) { + if (myUsage == 0 || totalUsage == 0) { if (log.isDebugEnabled()) { - log.debug("computeLocalQuota: totalUsage is zero; " - + "returning the configured usage ({}) as new local quota", + log.debug("computeLocalQuota: totalUsage or myUsage is zero; " + + "returning the configured usage ({}) as new local quota", confUsage); } return confUsage; @@ -72,16 +81,25 @@ public long computeLocalQuota(long confUsage, long myUsage, long[] allUsages) th log.warn(errMesg); } - // How much unused capacity is left over? - float residual = confUsage - totalUsage; + int resourceUsageTransportPublishIntervalInSecs = + pulsarService.getConfiguration().getResourceUsageTransportPublishIntervalInSecs(); + double resourceGroupLocalQuotaThreshold = + pulsarService.getConfiguration().getResourceGroupLocalQuotaThreshold(); + float totalRate = (float) totalUsage / resourceUsageTransportPublishIntervalInSecs; + double adjustedConfUsage = confUsage * resourceGroupLocalQuotaThreshold; + if (totalRate <= adjustedConfUsage) { + log.info("computeLocalQuota: total usage ({}) is less than the " + + "configured usage * threshold ({}), " + + "returning the configured usage ({}) as new local quota", + totalRate, adjustedConfUsage, confUsage); + return confUsage; + } // New quota is the old usage incremented by any residual as a ratio of the local usage to the total usage. // This should result in the calculatedQuota increasing proportionately if total usage is less than the // configured usage, and reducing proportionately if the total usage is greater than the configured usage. - // Capped to 1, to prevent negative or zero setting of quota. - // the rate limiter code assumes that rate value of 0 or less to mean that no rate limit should be applied float myUsageFraction = (float) myUsage / totalUsage; - float calculatedQuota = max(myUsage + residual * myUsageFraction, 1); + float calculatedQuota = Math.max(myUsageFraction * confUsage, 1); val longCalculatedQuota = (long) calculatedQuota; if (log.isDebugEnabled()) { @@ -95,34 +113,50 @@ public long computeLocalQuota(long confUsage, long myUsage, long[] allUsages) th @Override // Return true if a report needs to be sent for the current round; false if it can be suppressed for this round. public boolean needToReportLocalUsage(long currentBytesUsed, long lastReportedBytes, - long currentMessagesUsed, long lastReportedMessages, - long lastReportTimeMSecsSinceEpoch) { - // If we are about to go more than maxUsageReportSuppressRounds without reporting, send a report. - long currentTimeMSecs = System.currentTimeMillis(); - long mSecsSinceLastReport = currentTimeMSecs - lastReportTimeMSecsSinceEpoch; - if (mSecsSinceLastReport >= ResourceGroupService.maxIntervalForSuppressingReportsMSecs) { + long currentMessagesUsed, long lastReportedMessages, + long lastReportTimeMSecsSinceEpoch) { + ServiceConfiguration configuration = pulsarService.getConfiguration(); + long resourceUsageMaxUsageReportSuppressRounds = configuration.getResourceUsageMaxUsageReportSuppressRounds(); + if (resourceUsageMaxUsageReportSuppressRounds == 0) { return true; + } else if (resourceUsageMaxUsageReportSuppressRounds > 0) { + // If we are about to go more than maxUsageReportSuppressRounds without reporting, send a report. + long currentTimeMSecs = System.currentTimeMillis(); + long mSecsSinceLastReport = currentTimeMSecs - lastReportTimeMSecsSinceEpoch; + if (mSecsSinceLastReport >= TimeUnit.SECONDS.toMillis( + (long) configuration.getResourceUsageTransportPublishIntervalInSecs() + * configuration.getResourceUsageMaxUsageReportSuppressRounds())) { + return true; + } } // If the percentage change (increase or decrease) in usage is more than a threshold for // either bytes or messages, send a report. - final float toleratedDriftPercentage = ResourceGroupService.UsageReportSuppressionTolerancePercentage; - if (currentBytesUsed > 0) { - long diff = abs(currentBytesUsed - lastReportedBytes); - float diffPercentage = (float) diff * 100 / lastReportedBytes; - if (diffPercentage > toleratedDriftPercentage) { - return true; - } + final float toleratedDriftPercentage = configuration.getResourceUsageReportSuppressionTolerancePercentage(); + + if (needToReportLocalUsage(currentBytesUsed, lastReportedBytes, toleratedDriftPercentage)) { + return true; } - if (currentMessagesUsed > 0) { - long diff = abs(currentMessagesUsed - lastReportedMessages); - float diffPercentage = (float) diff * 100 / lastReportedMessages; - if (diffPercentage > toleratedDriftPercentage) { + if (needToReportLocalUsage(currentMessagesUsed, lastReportedMessages, toleratedDriftPercentage)) { + return true; + } + + return false; + } + + private boolean needToReportLocalUsage(long currentUsed, long lastReported, float toleratedDriftPercentage) { + if (currentUsed > 0) { + if (lastReported == 0) { return true; } + if (toleratedDriftPercentage <= 0) { + return true; + } + long diff = abs(currentUsed - lastReported); + float diffPercentage = (float) diff * 100 / lastReported; + return diffPercentage > toleratedDriftPercentage; } - return false; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java index f074f234b873f..7a27c0aed6378 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java @@ -20,6 +20,7 @@ import static org.apache.bookkeeper.mledger.util.PositionAckSetUtil.andAckSet; import static org.apache.bookkeeper.mledger.util.PositionAckSetUtil.isAckSetEmpty; +import static org.apache.pulsar.broker.service.persistent.DispatchRateLimiter.Type.RESOURCE_GROUP; import static org.apache.pulsar.broker.service.persistent.PersistentTopic.MESSAGE_RATE_BACKOFF_MS; import io.netty.buffer.ByteBuf; import io.prometheus.client.Gauge; @@ -40,6 +41,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.intercept.BrokerInterceptor; +import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter; import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -382,7 +384,8 @@ public void resetCloseFuture() { protected boolean hasAnyDispatchRateLimiter() { return subscription.getTopic().getBrokerDispatchRateLimiter().isPresent() || subscription.getTopic().getDispatchRateLimiter().isPresent() - || getRateLimiter().isPresent(); + || getRateLimiter().isPresent() + || subscription.getTopic().getResourceGroupDispatchRateLimiter().isPresent(); } protected Pair applyRateLimitsToMessagesAndBytesToRead(int messagesToRead, long bytesToRead) { @@ -406,6 +409,10 @@ protected Pair applyRateLimitsToMessagesAndBytesToRead(int messag DispatchRateLimiter.Type.SUBSCRIPTION); } + if (success) { + success = applyDispatchRateLimitsToReadLimits(null, readLimits, RESOURCE_GROUP); + } + return readLimits; } @@ -414,29 +421,47 @@ private boolean applyDispatchRateLimitsToReadLimits(DispatchRateLimiter rateLimi DispatchRateLimiter.Type limiterType) { int originalMessagesToRead = readLimits.getLeft(); long originalBytesToRead = readLimits.getRight(); - // update messagesToRead according to available dispatch rate limit. - int availablePermitsOnMsg = (int) rateLimiter.getAvailableDispatchRateLimitOnMsg(); - if (availablePermitsOnMsg >= 0) { - readLimits.setLeft(Math.min(readLimits.getLeft(), availablePermitsOnMsg)); - } - long availablePermitsOnByte = rateLimiter.getAvailableDispatchRateLimitOnByte(); - if (availablePermitsOnByte >= 0) { - readLimits.setRight(Math.min(readLimits.getRight(), availablePermitsOnByte)); - } - if (readLimits.getLeft() < originalMessagesToRead) { - switch (limiterType) { - case BROKER -> dispatchThrottledMsgEventsByBrokerLimit.increment(); - case TOPIC -> dispatchThrottledMsgEventsByTopicLimit.increment(); - case SUBSCRIPTION -> dispatchThrottledMsgEventsBySubscriptionLimit.increment(); - default -> {} + if (RESOURCE_GROUP.equals(limiterType)) { + ResourceGroupDispatchLimiter resourceGroupDispatchLimiter = + subscription.getTopic().getResourceGroupDispatchRateLimiter().orElse(null); + if (resourceGroupDispatchLimiter == null) { + return true; } - } - if (readLimits.getRight() < originalBytesToRead) { - switch (limiterType) { - case BROKER -> dispatchThrottledBytesEventsByBrokerLimit.increment(); - case TOPIC -> dispatchThrottledBytesEventsByTopicLimit.increment(); - case SUBSCRIPTION -> dispatchThrottledBytesEventsBySubscriptionLimit.increment(); - default -> {} + long availablePermitsOnMsgByRG = resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnMsg(); + if (availablePermitsOnMsgByRG >= 0) { + readLimits.setLeft((int) Math.min(readLimits.getLeft(), availablePermitsOnMsgByRG)); + } + long availablePermitsOnByteByRG = resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnByte(); + if (availablePermitsOnByteByRG >= 0) { + readLimits.setRight(Math.min(readLimits.getRight(), availablePermitsOnByteByRG)); + } + } else { + // update messagesToRead according to available dispatch rate limit. + int availablePermitsOnMsg = (int) rateLimiter.getAvailableDispatchRateLimitOnMsg(); + if (availablePermitsOnMsg >= 0) { + readLimits.setLeft(Math.min(readLimits.getLeft(), availablePermitsOnMsg)); + } + long availablePermitsOnByte = rateLimiter.getAvailableDispatchRateLimitOnByte(); + if (availablePermitsOnByte >= 0) { + readLimits.setRight(Math.min(readLimits.getRight(), availablePermitsOnByte)); + } + if (readLimits.getLeft() < originalMessagesToRead) { + switch (limiterType) { + case BROKER -> dispatchThrottledMsgEventsByBrokerLimit.increment(); + case TOPIC -> dispatchThrottledMsgEventsByTopicLimit.increment(); + case SUBSCRIPTION -> dispatchThrottledMsgEventsBySubscriptionLimit.increment(); + default -> { + } + } + } + if (readLimits.getRight() < originalBytesToRead) { + switch (limiterType) { + case BROKER -> dispatchThrottledBytesEventsByBrokerLimit.increment(); + case TOPIC -> dispatchThrottledBytesEventsByTopicLimit.increment(); + case SUBSCRIPTION -> dispatchThrottledBytesEventsBySubscriptionLimit.increment(); + default -> { + } + } } } if (readLimits.getLeft() == 0 || readLimits.getRight() == 0) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index e0b310712d13f..648d97992aacc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -31,6 +31,7 @@ import java.util.Collection; import java.util.Collections; import java.util.EnumSet; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -58,12 +59,15 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.resourcegroup.ResourceGroup; +import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter; import org.apache.pulsar.broker.resourcegroup.ResourceGroupPublishLimiter; +import org.apache.pulsar.broker.resourcegroup.ResourceGroupService; import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException; import org.apache.pulsar.broker.service.BrokerServiceException.ProducerBusyException; import org.apache.pulsar.broker.service.BrokerServiceException.ProducerFencedException; import org.apache.pulsar.broker.service.BrokerServiceException.TopicMigratedException; import org.apache.pulsar.broker.service.BrokerServiceException.TopicTerminatedException; +import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter; import org.apache.pulsar.broker.service.plugin.EntryFilter; import org.apache.pulsar.broker.service.schema.SchemaRegistryService; import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; @@ -79,6 +83,7 @@ import org.apache.pulsar.common.policies.data.HierarchyTopicPolicies; import org.apache.pulsar.common.policies.data.InactiveTopicPolicies; import org.apache.pulsar.common.policies.data.Policies; +import org.apache.pulsar.common.policies.data.PolicyHierarchyValue; import org.apache.pulsar.common.policies.data.PublishRate; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; @@ -140,6 +145,11 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener { protected volatile PublishRateLimiter topicPublishRateLimiter; protected volatile ResourceGroupPublishLimiter resourceGroupPublishLimiter; + @Getter + protected volatile Optional resourceGroupDispatchRateLimiter = Optional.empty(); + + protected boolean preciseTopicPublishRateLimitingEnable; + @Getter protected boolean resourceGroupRateLimitingEnabled; @@ -235,8 +245,36 @@ public List getEntryFilters() { return this.entryFilters.getRight(); } - public DispatchRateImpl getReplicatorDispatchRate() { - return this.topicPolicies.getReplicatorDispatchRate().get(); + public DispatchRateImpl getReplicatorDispatchRate(String remoteCluster) { + String localCluster = brokerService.pulsar().getConfiguration().getClusterName(); + PolicyHierarchyValue dispatchRatePolicyHierarchyValue = + topicPolicies.getReplicatorDispatchRate() + .get(DispatchRateLimiter.getReplicatorDispatchRateKey(localCluster, remoteCluster)); + DispatchRateImpl dispatchRate; + if (dispatchRatePolicyHierarchyValue != null) { + // Topic + dispatchRate = dispatchRatePolicyHierarchyValue.getTopicValue(); + if (dispatchRate == null) { + dispatchRate = topicPolicies.getReplicatorDispatchRate().get(localCluster).getTopicValue(); + } + // Namespace + if (dispatchRate == null) { + dispatchRate = dispatchRatePolicyHierarchyValue.getNamespaceValue(); + } + if (dispatchRate == null) { + dispatchRate = topicPolicies.getReplicatorDispatchRate().get(localCluster).getNamespaceValue(); + } + // Broker + if (dispatchRate == null) { + dispatchRate = dispatchRatePolicyHierarchyValue.getBrokerValue(); + } + if (dispatchRate == null) { + dispatchRate = topicPolicies.getReplicatorDispatchRate().get(localCluster).getBrokerValue(); + } + } else { + dispatchRate = topicPolicies.getReplicatorDispatchRate().get(localCluster).get(); + } + return dispatchRate; } private SchemaCompatibilityStrategy formatSchemaCompatibilityStrategy(SchemaCompatibilityStrategy strategy) { @@ -285,8 +323,9 @@ protected void updateTopicPolicy(TopicPolicies data) { isGlobalPolicies); topicPolicies.getPublishRate().updateTopicValue(PublishRate.normalize(data.getPublishRate()), isGlobalPolicies); topicPolicies.getDelayedDeliveryEnabled().updateTopicValue(data.getDelayedDeliveryEnabled(), isGlobalPolicies); - topicPolicies.getReplicatorDispatchRate().updateTopicValue( - DispatchRateImpl.normalize(data.getReplicatorDispatchRate()), isGlobalPolicies); + updateTopicLevelReplicatorDispatchRate( + data.getReplicatorDispatchRateMap() != null ? data.getReplicatorDispatchRateMap() : new HashMap<>(), + data.getReplicatorDispatchRate()); topicPolicies.getDelayedDeliveryTickTimeMillis().updateTopicValue(data.getDelayedDeliveryTickTimeMillis(), isGlobalPolicies); topicPolicies.getDelayedDeliveryMaxDelayInMillis().updateTopicValue(data.getDelayedDeliveryMaxDelayInMillis(), @@ -304,10 +343,43 @@ protected void updateTopicPolicy(TopicPolicies data) { topicPolicies.getDispatcherPauseOnAckStatePersistentEnabled() .updateTopicValue(data.getDispatcherPauseOnAckStatePersistentEnabled(), isGlobalPolicies); this.subscriptionPolicies = data.getSubscriptionPolicies(); + topicPolicies.getResourceGroupName().updateTopicValue(data.getResourceGroupName()); updateEntryFilters(); } + private void updateTopicLevelReplicatorDispatchRate(Map policy, + DispatchRateImpl defaultDispatchRate) { + Map dispatchRateMap = new HashMap<>(); + if (policy != null) { + dispatchRateMap.putAll(policy); + } + + // Backward compatibility: Default use the current cluster name as key. + dispatchRateMap.putIfAbsent(brokerService.pulsar().getConfiguration().getClusterName(), defaultDispatchRate); + + // Process existing topic policies + topicPolicies.getReplicatorDispatchRate().forEach((cluster, policyValue) -> { + DispatchRateImpl dispatchRate = dispatchRateMap.remove(cluster); + policyValue.updateTopicValue(dispatchRate); + }); + + // Process remaining that weren't in topic policies + dispatchRateMap.forEach((cluster, dispatchRate) -> + updateReplicatorDispatchRate(cluster, value -> value.updateTopicValue(dispatchRate))); + } + + private void updateReplicatorDispatchRate(String cluster, + java.util.function.Consumer> c) { + topicPolicies.getReplicatorDispatchRate().compute(cluster, (k, v) -> { + if (v == null) { + v = new PolicyHierarchyValue<>(); + } + c.accept(v); + return v; + }); + } + protected void updateTopicPolicyByNamespacePolicy(Policies namespacePolicies) { if (log.isDebugEnabled()) { log.debug("[{}]updateTopicPolicyByNamespacePolicy,data={}", topic, namespacePolicies); @@ -351,8 +423,7 @@ protected void updateTopicPolicyByNamespacePolicy(Policies namespacePolicies) { .map(DelayedDeliveryPolicies::getMaxDeliveryDelayInMillis).orElse(null)); topicPolicies.getSubscriptionTypesEnabled().updateNamespaceValue( subTypeStringsToEnumSet(namespacePolicies.subscription_types_enabled)); - updateNamespaceReplicatorDispatchRate(namespacePolicies, - brokerService.getPulsar().getConfig().getClusterName()); + updateNamespaceLevelReplicatorDispatchRate(namespacePolicies.replicatorDispatchRate); Arrays.stream(BacklogQuota.BacklogQuotaType.values()).forEach( type -> this.topicPolicies.getBackLogQuotaMap().get(type) .updateNamespaceValue(MapUtils.getObject(namespacePolicies.backlog_quota_map, type))); @@ -366,6 +437,8 @@ protected void updateTopicPolicyByNamespacePolicy(Policies namespacePolicies) { topicPolicies.getDispatcherPauseOnAckStatePersistentEnabled().updateNamespaceValue( namespacePolicies.dispatcherPauseOnAckStatePersistentEnabled); + topicPolicies.getResourceGroupName().updateNamespaceValue(namespacePolicies.resource_group_name); + updateEntryFilters(); } @@ -373,6 +446,23 @@ private Integer normalizeValue(Integer policyValue) { return policyValue != null && policyValue < 0 ? null : policyValue; } + private void updateNamespaceLevelReplicatorDispatchRate(Map policy) { + Map dispatchRateMap = new HashMap<>(); + if (policy != null) { + dispatchRateMap.putAll(policy); + } + + // Process existing topic policies. + topicPolicies.getReplicatorDispatchRate().forEach((cluster, policyValue) -> { + DispatchRateImpl dispatchRate = dispatchRateMap.remove(cluster); + policyValue.updateNamespaceValue(dispatchRate); + }); + + // Process remaining that weren't in topic policies + dispatchRateMap.forEach((cluster, dispatchRate) -> updateReplicatorDispatchRate(cluster, + value -> value.updateNamespaceValue(dispatchRate))); + } + private void updateNamespaceDispatchRate(Policies namespacePolicies, String cluster) { DispatchRateImpl dispatchRate = namespacePolicies.topicDispatchRate.get(cluster); if (dispatchRate == null) { @@ -391,11 +481,6 @@ private void updateNamespaceSubscriptionDispatchRate(Policies namespacePolicies, .updateNamespaceValue(DispatchRateImpl.normalize(namespacePolicies.subscriptionDispatchRate.get(cluster))); } - private void updateNamespaceReplicatorDispatchRate(Policies namespacePolicies, String cluster) { - topicPolicies.getReplicatorDispatchRate() - .updateNamespaceValue(DispatchRateImpl.normalize(namespacePolicies.replicatorDispatchRate.get(cluster))); - } - private void updateSchemaCompatibilityStrategyNamespaceValue(Policies namespacePolicies){ if (isSystemTopic()) { return; @@ -456,7 +541,7 @@ private void updateTopicPolicyByBrokerConfig() { topicPolicies.getCompactionThreshold().updateBrokerValue(config.getBrokerServiceCompactionThresholdInBytes()); topicPolicies.getReplicationClusters().updateBrokerValue(Collections.emptyList()); SchemaCompatibilityStrategy schemaCompatibilityStrategy = config.getSchemaCompatibilityStrategy(); - topicPolicies.getReplicatorDispatchRate().updateBrokerValue(replicatorDispatchRateInBroker(config)); + updateBrokerReplicatorDispatchRate(); if (isSystemTopic()) { schemaCompatibilityStrategy = config.getSystemTopicSchemaCompatibilityStrategy(); } @@ -1143,25 +1228,50 @@ public void updateResourceGroupLimiter(Optional optPolicies) { public void updateResourceGroupLimiter(@NonNull Policies namespacePolicies) { requireNonNull(namespacePolicies); - // attach the resource-group level rate limiters, if set - String rgName = namespacePolicies.resource_group_name; + topicPolicies.getResourceGroupName().updateNamespaceValue(namespacePolicies.resource_group_name); + updateResourceGroupLimiter(); + } + + public void updateResourceGroupLimiter() { + String rgName = topicPolicies.getResourceGroupName().get(); if (rgName != null) { - final ResourceGroup resourceGroup = - brokerService.getPulsar().getResourceGroupServiceManager().resourceGroupGet(rgName); + ResourceGroupService resourceGroupService = brokerService.getPulsar().getResourceGroupServiceManager(); + final ResourceGroup resourceGroup = resourceGroupService.resourceGroupGet(rgName); if (resourceGroup != null) { + TopicName topicName = TopicName.get(topic); + resourceGroupService.unRegisterTopic(topicName); + String topicRg = topicPolicies.getResourceGroupName().getTopicValue(); + if (topicRg != null) { + try { + resourceGroupService.registerTopic(topicRg, topicName); + } catch (Exception e) { + log.error("Failed to register resource group {} for topic {}", rgName, topic); + return; + } + } this.resourceGroupRateLimitingEnabled = true; this.resourceGroupPublishLimiter = resourceGroup.getResourceGroupPublishLimiter(); + this.resourceGroupDispatchRateLimiter = Optional.of(resourceGroup.getResourceGroupDispatchLimiter()); log.info("Using resource group {} rate limiter for topic {}", rgName, topic); } } else { - if (this.resourceGroupRateLimitingEnabled) { - this.resourceGroupPublishLimiter = null; - this.resourceGroupRateLimitingEnabled = false; - } + closeResourceGroupLimiter(); } updateActiveRateLimiters(); } + protected void closeResourceGroupLimiter() { + if (resourceGroupRateLimitingEnabled) { + this.resourceGroupPublishLimiter = null; + this.resourceGroupDispatchRateLimiter = Optional.empty(); + this.resourceGroupRateLimitingEnabled = false; + } + ResourceGroupService resourceGroupServiceManager = brokerService.getPulsar().getResourceGroupServiceManager(); + if (resourceGroupServiceManager != null) { + resourceGroupServiceManager.unRegisterTopic(TopicName.get(topic)); + } + } + public void updateEntryFilters() { if (isSystemTopic()) { entryFilters = Pair.of(null, Collections.emptyList()); @@ -1301,8 +1411,9 @@ public void updateBrokerSubscriptionDispatchRate() { } public void updateBrokerReplicatorDispatchRate() { - topicPolicies.getReplicatorDispatchRate().updateBrokerValue( - replicatorDispatchRateInBroker(brokerService.pulsar().getConfiguration())); + updateReplicatorDispatchRate(brokerService.pulsar().getConfiguration().getClusterName(), v -> { + v.updateBrokerValue(replicatorDispatchRateInBroker(brokerService.pulsar().getConfiguration())); + }); } public void updateBrokerDispatchRate() { @@ -1388,4 +1499,11 @@ public boolean isSystemCursor(String sub) { return COMPACTION_SUBSCRIPTION.equals(sub) || (additionalSystemCursorNames != null && additionalSystemCursorNames.contains(sub)); } + + public static String getReplicatorDispatchRateKey(String localCluster, String remoteCluster) { + if (StringUtils.isNotEmpty(remoteCluster)) { + return String.format("%s->%s", remoteCluster, localCluster); + } + return localCluster; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java index 86e2b6e74de89..64d03f6aa95dd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java @@ -20,6 +20,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; +import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter; import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter; import org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl; @@ -50,6 +51,10 @@ default Optional getRateLimiter() { return Optional.empty(); } + default Optional getResourceGroupDispatchRateLimiter() { + return Optional.empty(); + } + boolean isConnected(); long getNumberOfEntriesInBacklog(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java index 8f66d9c0e3e0f..274e3211853b4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java @@ -25,6 +25,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.mledger.Position; +import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter; import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter; import org.apache.pulsar.broker.service.persistent.SubscribeRateLimiter; import org.apache.pulsar.broker.service.plugin.EntryFilter; @@ -330,6 +331,10 @@ default Optional getDispatchRateLimiter() { return Optional.empty(); } + default Optional getResourceGroupDispatchRateLimiter() { + return Optional.empty(); + } + default Optional getSubscribeRateLimiter() { return Optional.empty(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 1931a09497e3f..e5c08b40bfc07 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -177,7 +177,7 @@ public CompletableFuture initialize() { isAllowAutoUpdateSchema = policies.is_allow_auto_update_schema; } updatePublishRateLimiter(); - updateResourceGroupLimiter(policies); + updateResourceGroupLimiter(); return updateClusterMigrated(); }); } @@ -457,6 +457,7 @@ private CompletableFuture delete(boolean failIfHasSubscriptions, boolean c brokerService.executor().execute(() -> { brokerService.removeTopicFromCache(NonPersistentTopic.this); unregisterTopicPolicyListener(); + closeResourceGroupLimiter(); log.info("[{}] Topic deleted", topic); deleteFuture.complete(null); }); @@ -558,6 +559,7 @@ public CompletableFuture close( if (disconnectClients) { brokerService.removeTopicFromCache(NonPersistentTopic.this); unregisterTopicPolicyListener(); + closeResourceGroupLimiter(); } closeFuture.complete(null); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java index b78af75a5169e..2a94b18e85fd5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java @@ -20,6 +20,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; +import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.common.naming.NamespaceName; @@ -30,11 +31,20 @@ import org.slf4j.LoggerFactory; public abstract class DispatchRateLimiter { + + public static String getReplicatorDispatchRateKey(String localCluster, String remoteCluster) { + if (StringUtils.isNotEmpty(remoteCluster)) { + return String.format("%s->%s", localCluster, remoteCluster); + } + return localCluster; + } + public enum Type { TOPIC, SUBSCRIPTION, REPLICATOR, - BROKER + BROKER, + RESOURCE_GROUP } protected final PersistentTopic topic; @@ -147,7 +157,7 @@ public final void updateDispatchRate() { dispatchRate = topic.getSubscriptionDispatchRate(subscriptionName); break; case REPLICATOR: - dispatchRate = topic.getReplicatorDispatchRate(); + dispatchRate = topic.getReplicatorDispatchRate(subscriptionName); break; case BROKER: dispatchRate = createDispatchRate(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java index 46f8a27d58020..49260f9b6a06a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java @@ -186,6 +186,8 @@ protected boolean replicateEntries(List entries, final InFlightTask inFli } dispatchRateLimiter.ifPresent(rateLimiter -> rateLimiter.consumeDispatchQuota(1, entry.getLength())); + resourceGroupDispatchRateLimiter.ifPresent(rateLimiter -> rateLimiter.tryAcquire(1, entry.getLength())); + msg.setReplicatedFrom(localCluster); headersAndPayload.retain(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index e0a31476fc9f7..c37bb3b579617 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -35,6 +35,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import lombok.AllArgsConstructor; import lombok.Data; import lombok.Getter; @@ -53,6 +54,8 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.resourcegroup.ResourceGroup; +import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter; import org.apache.pulsar.broker.service.AbstractReplicator; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.BrokerServiceException; @@ -84,6 +87,9 @@ public abstract class PersistentReplicator extends AbstractReplicator protected final String localSchemaTopicName; protected Optional dispatchRateLimiter = Optional.empty(); + protected volatile Optional resourceGroupDispatchRateLimiter = Optional.empty(); + private final Consumer resourceGroupDispatchRateLimiterConsumer = (v) -> + resourceGroupDispatchRateLimiter = Optional.ofNullable(v); private final Object dispatchRateLimiterLock = new Object(); private int readBatchSize; @@ -271,6 +277,33 @@ private AvailablePermits getRateLimiterAvailablePermits(int availablePermits) { } } + if (resourceGroupDispatchRateLimiter.isPresent()) { + ResourceGroupDispatchLimiter rateLimiter = resourceGroupDispatchRateLimiter.get(); + long rgAvailablePermitsOnMsg = rateLimiter.getAvailableDispatchRateLimitOnMsg(); + long rgAvailablePermitsOnByte = rateLimiter.getAvailableDispatchRateLimitOnByte(); + if (rgAvailablePermitsOnMsg == 0 || rgAvailablePermitsOnByte == 0) { + if (log.isDebugEnabled()) { + log.debug("[{}] message-read exceeded resourcegroup message-rate {}/{}," + + " schedule after a {}", + replicatorId, + rateLimiter.getDispatchRateOnMsg(), + rateLimiter.getDispatchRateOnByte(), + MESSAGE_RATE_BACKOFF_MS); + } + return new AvailablePermits(-1, -1); + } + if (availablePermitsOnMsg == -1) { + availablePermitsOnMsg = rgAvailablePermitsOnMsg; + } else { + availablePermitsOnMsg = Math.min(rgAvailablePermitsOnMsg, availablePermitsOnMsg); + } + if (availablePermitsOnByte == -1) { + availablePermitsOnByte = rgAvailablePermitsOnByte; + } else { + availablePermitsOnByte = Math.min(rgAvailablePermitsOnByte, availablePermitsOnByte); + } + } + availablePermitsOnMsg = availablePermitsOnMsg == -1 ? availablePermits : Math.min(availablePermits, availablePermitsOnMsg); availablePermitsOnMsg = Math.min(availablePermitsOnMsg, readBatchSize); @@ -629,7 +662,7 @@ public void deleteFailed(ManagedLedgerException exception, Object ctx) { exception.getMessage(), exception); if (exception instanceof CursorAlreadyClosedException) { log.warn("[{}] Asynchronous ack failure because replicator is already deleted and cursor is already" - + " closed {}, ({})", replicatorId, ctx, exception.getMessage(), exception); + + " closed {}, ({})", replicatorId, ctx, exception.getMessage(), exception); // replicator is already deleted and cursor is already closed so, producer should also be disconnected. terminate(); return; @@ -714,18 +747,55 @@ public Optional getRateLimiter() { return dispatchRateLimiter; } + @Override + public Optional getResourceGroupDispatchRateLimiter() { + return resourceGroupDispatchRateLimiter; + } + + // This is used to unregister the resource group dispatch rate limiter + private String usedResourceGroupName; + @Override public void initializeDispatchRateLimiterIfNeeded() { synchronized (dispatchRateLimiterLock) { if (!dispatchRateLimiter.isPresent() - && DispatchRateLimiter.isDispatchRateEnabled(topic.getReplicatorDispatchRate())) { - this.dispatchRateLimiter = Optional.of( - topic.getBrokerService().getDispatchRateLimiterFactory() - .createReplicatorDispatchRateLimiter(topic, Codec.decode(cursor.getName()))); + && DispatchRateLimiter.isDispatchRateEnabled(topic.getReplicatorDispatchRate(remoteCluster))) { + this.dispatchRateLimiter = Optional.of(topic.getBrokerService().getDispatchRateLimiterFactory() + .createReplicatorDispatchRateLimiter(topic, remoteCluster)); + } + + String resourceGroupName = topic.getHierarchyTopicPolicies().getResourceGroupName().get(); + if (resourceGroupName != null) { + if (usedResourceGroupName != null) { + unregisterReplicatorDispatchRateLimiter(); + } + usedResourceGroupName = resourceGroupName; + ResourceGroup resourceGroup = + brokerService.getPulsar().getResourceGroupServiceManager().resourceGroupGet(resourceGroupName); + if (resourceGroup != null) { + resourceGroup.registerReplicatorDispatchRateLimiter(remoteCluster, + resourceGroupDispatchRateLimiterConsumer); + } + } else { + unregisterReplicatorDispatchRateLimiter(); } } } + private void unregisterReplicatorDispatchRateLimiter() { + if (usedResourceGroupName == null) { + return; + } + ResourceGroup resourceGroup = + brokerService.getPulsar().getResourceGroupServiceManager() + .resourceGroupGet(usedResourceGroupName); + if (resourceGroup != null) { + resourceGroup.unregisterReplicatorDispatchRateLimiter(remoteCluster, + resourceGroupDispatchRateLimiterConsumer); + } + usedResourceGroupName = null; + } + @Override public void updateRateLimiter() { initializeDispatchRateLimiterIfNeeded(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 101bf30de96fa..09041571e7bf5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -472,7 +472,7 @@ public CompletableFuture initialize() { if (!optPolicies.isPresent()) { isEncryptionRequired = false; updatePublishRateLimiter(); - updateResourceGroupLimiter(new Policies()); + updateResourceGroupLimiter(); initializeDispatchRateLimiterIfNeeded(); updateSubscribeRateLimiter(); return; @@ -488,7 +488,7 @@ public CompletableFuture initialize() { updatePublishRateLimiter(); - updateResourceGroupLimiter(policies); + updateResourceGroupLimiter(); this.isEncryptionRequired = policies.encryption_required; @@ -1598,6 +1598,8 @@ public void deleteLedgerComplete(Object ctx) { unregisterTopicPolicyListener(); + closeResourceGroupLimiter(); + log.info("[{}] Topic deleted", topic); deleteFuture.complete(null); } @@ -1876,6 +1878,9 @@ private void disposeTopic(CompletableFuture closeFuture) { subscribeRateLimiter.ifPresent(SubscribeRateLimiter::close); unregisterTopicPolicyListener(); + + closeResourceGroupLimiter(); + log.info("[{}] Topic closed", topic); cancelFencedTopicMonitoringTask(); closeFuture.complete(null); @@ -3673,7 +3678,7 @@ public CompletableFuture onPoliciesUpdate(@NonNull Policies data) { // Apply policies for components. List> applyPolicyTasks = applyUpdatedTopicPolicies(); - applyPolicyTasks.add(applyUpdatedNamespacePolicies(data)); + applyPolicyTasks.add(applyUpdatedNamespacePolicies()); return FutureUtil.waitForAll(applyPolicyTasks) .thenAccept(__ -> log.info("[{}] namespace-level policies updated successfully", topic)) .exceptionally(ex -> { @@ -3682,8 +3687,8 @@ public CompletableFuture onPoliciesUpdate(@NonNull Policies data) { }); } - private CompletableFuture applyUpdatedNamespacePolicies(Policies namespaceLevelPolicies) { - return FutureUtil.runWithCurrentThread(() -> updateResourceGroupLimiter(namespaceLevelPolicies)); + private CompletableFuture applyUpdatedNamespacePolicies() { + return FutureUtil.runWithCurrentThread(() -> updateResourceGroupLimiter()); } private List> applyUpdatedTopicPolicies() { @@ -3702,6 +3707,7 @@ private List> applyUpdatedTopicPolicies() { applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> updateDispatchRateLimiter())); applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> updateSubscribeRateLimiter())); applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> updatePublishRateLimiter())); + applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> updateResourceGroupLimiter())); applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> updateSubscriptionsDispatcherRateLimiter())); applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread( diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java index a334fd86dd02f..a5d0c6216f6d4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java @@ -114,6 +114,7 @@ protected boolean replicateEntries(List entries, InFlightTask inFlightTas } dispatchRateLimiter.ifPresent(rateLimiter -> rateLimiter.consumeDispatchQuota(1, entry.getLength())); + resourceGroupDispatchRateLimiter.ifPresent(rateLimiter -> rateLimiter.tryAcquire(1, entry.getLength())); msgOut.recordEvent(msg.getDataBuffer().readableBytes()); stats.incrementMsgOutCounter(); diff --git a/pulsar-broker/src/main/proto/ResourceUsage.proto b/pulsar-broker/src/main/proto/ResourceUsage.proto index 4706c9dfbcd19..91559c8474695 100644 --- a/pulsar-broker/src/main/proto/ResourceUsage.proto +++ b/pulsar-broker/src/main/proto/ResourceUsage.proto @@ -31,6 +31,12 @@ message StorageUsage { required uint64 totalBytes = 1; } +message ReplicatorUsage { + required string localCluster = 1; + required string remoteCluster = 2; + required NetworkUsage networkUsage = 3; +} + message ResourceUsage { // owner is the key(ID) of the entity that reports the usage // It could be a resource-group or tenant, namespace or a topic @@ -38,6 +44,9 @@ message ResourceUsage { optional NetworkUsage publish = 2; optional NetworkUsage dispatch = 3; optional StorageUsage storage = 4; + optional NetworkUsage replicationDispatch = 6 [deprecated = true]; + // more detailed replicator usage per cluster pair + repeated ReplicatorUsage replicator = 7; } message ResourceUsageInfo { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminReplicatorDispatchRateTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminReplicatorDispatchRateTest.java new file mode 100644 index 0000000000000..ce647c2a398b5 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminReplicatorDispatchRateTest.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.admin; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.DispatchRate; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +public class AdminReplicatorDispatchRateTest extends MockedPulsarServiceBaseTest { + @BeforeClass + @Override + public void setup() throws Exception { + super.internalSetup(); + setupDefaultTenantAndNamespace(); + } + + @AfterClass(alwaysRun = true) + @Override + public void cleanup() throws Exception { + super.internalCleanup(); + } + + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + conf.setTopicLevelPoliciesEnabled(true); + conf.setSystemTopicEnabled(true); + } + + @Test + public void testReplicatorDispatchRateOnTopicLevel() throws Exception { + TopicName topicName = TopicName.get("test-replicator-dispatch-rate"); + String topic = topicName.toString(); + + admin.topics().createNonPartitionedTopic(topic); + + Awaitility.await().untilAsserted(() -> { + assertNull(admin.topicPolicies().getReplicatorDispatchRate(topic, false)); + // The broker config is used when the replicator dispatch rate is not set. + assertReplicatorDispatchRateByBrokerConfig(admin.topicPolicies().getReplicatorDispatchRate(topic, true)); + }); + + // Set the default replicator dispatch rate on the topic level. + DispatchRate defaultDispatchRate = DispatchRate.builder() + .dispatchThrottlingRateInMsg(100) + .dispatchThrottlingRateInByte(200) + .ratePeriodInSecond(1) + .build(); + admin.topicPolicies().setReplicatorDispatchRate(topic, defaultDispatchRate); + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topicPolicies().getReplicatorDispatchRate(topic, false), defaultDispatchRate); + assertEquals(admin.topicPolicies().getReplicatorDispatchRate(topic, true), defaultDispatchRate); + }); + + // Set the replicator dispatch rate for the r1 cluster on the topic leve. + String r1Cluster = "r1"; + DispatchRate r1DispatchRate = DispatchRate.builder() + .dispatchThrottlingRateInMsg(200) + .dispatchThrottlingRateInByte(400) + .ratePeriodInSecond(1) + .build(); + admin.topicPolicies().setReplicatorDispatchRate(topic, r1Cluster, r1DispatchRate); + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topicPolicies().getReplicatorDispatchRate(topic, true), defaultDispatchRate); + assertEquals(admin.topicPolicies().getReplicatorDispatchRate(topic, false), defaultDispatchRate); + assertEquals(admin.topicPolicies().getReplicatorDispatchRate(topic, r1Cluster, true), r1DispatchRate); + assertEquals(admin.topicPolicies().getReplicatorDispatchRate(topic, r1Cluster, false), r1DispatchRate); + }); + + // Remove the replicator dispatch rate for the r1 cluster on the topic level. + admin.topicPolicies().removeReplicatorDispatchRate(topic, r1Cluster); + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topicPolicies().getReplicatorDispatchRate(topic, true), defaultDispatchRate); + assertEquals(admin.topicPolicies().getReplicatorDispatchRate(topic, false), defaultDispatchRate); + assertNull(admin.topicPolicies().getReplicatorDispatchRate(topic, r1Cluster, false)); + assertEquals(admin.topicPolicies().getReplicatorDispatchRate(topic, r1Cluster, true), defaultDispatchRate); + }); + + // Remove the default replicator dispatch rate on the topic level. + admin.topicPolicies().removeReplicatorDispatchRate(topic); + Awaitility.await().untilAsserted(() -> { + assertNull(admin.topicPolicies().getReplicatorDispatchRate(topic, false)); + assertNull(admin.topicPolicies().getReplicatorDispatchRate(topic, r1Cluster, false)); + // The broker config is used when the replicator dispatch rate is not set on any level. + assertReplicatorDispatchRateByBrokerConfig( + admin.topicPolicies().getReplicatorDispatchRate(topic, r1Cluster, true)); + assertReplicatorDispatchRateByBrokerConfig(admin.topicPolicies().getReplicatorDispatchRate(topic, true)); + }); + } + + @Test + public void testReplicatorDispatchRateOnNamespaceAndTopicLevels() throws Exception { + String namespace = "public/test-replicator-dispatch-rate-ns"; + admin.namespaces().createNamespace(namespace); + + // Set the default replicator dispatch rate. + DispatchRate defaultDispatchRateOnNamespace = DispatchRate.builder() + .dispatchThrottlingRateInMsg(100) + .dispatchThrottlingRateInByte(200) + .ratePeriodInSecond(1) + .build(); + admin.namespaces().setReplicatorDispatchRate(namespace, defaultDispatchRateOnNamespace); + assertEquals(admin.namespaces().getReplicatorDispatchRate(namespace), defaultDispatchRateOnNamespace); + + // Set the replicator dispatch rate for the r1 cluster. + String r1Cluster = "r1"; + DispatchRate r1DispatchRateOnNamespace = DispatchRate.builder() + .dispatchThrottlingRateInMsg(200) + .dispatchThrottlingRateInByte(400) + .ratePeriodInSecond(1) + .build(); + admin.namespaces().setReplicatorDispatchRate(namespace, r1Cluster, r1DispatchRateOnNamespace); + assertEquals(admin.namespaces().getReplicatorDispatchRate(namespace), defaultDispatchRateOnNamespace); + assertEquals(admin.namespaces().getReplicatorDispatchRate(namespace, r1Cluster), r1DispatchRateOnNamespace); + + // Topic inherits the namespace level replicator dispatch rate. + String topic = TopicName.get(namespace + "/test-replicator-dispatch-rate").toString(); + admin.topics().createNonPartitionedTopic(topic); + assertNull(admin.topicPolicies().getReplicatorDispatchRate(topic)); + assertEquals(admin.topicPolicies().getReplicatorDispatchRate(topic, true), defaultDispatchRateOnNamespace); + assertEquals(admin.topicPolicies().getReplicatorDispatchRate(topic, r1Cluster, true), + r1DispatchRateOnNamespace); + + // Topic overrides the namespace level replicator dispatch rate. + DispatchRate defaultDispatchRateOnTopic = DispatchRate.builder() + .dispatchThrottlingRateInMsg(300) + .dispatchThrottlingRateInByte(600) + .ratePeriodInSecond(1) + .build(); + admin.topicPolicies().setReplicatorDispatchRate(topic, defaultDispatchRateOnTopic); + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topicPolicies().getReplicatorDispatchRate(topic, false), defaultDispatchRateOnTopic); + assertEquals(admin.topicPolicies().getReplicatorDispatchRate(topic, true), defaultDispatchRateOnTopic); + }); + + DispatchRate topicDispatchRateForR1 = DispatchRate.builder() + .dispatchThrottlingRateInMsg(400) + .dispatchThrottlingRateInByte(800) + .ratePeriodInSecond(1) + .build(); + admin.topicPolicies().setReplicatorDispatchRate(topic, r1Cluster, topicDispatchRateForR1); + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topicPolicies().getReplicatorDispatchRate(topic, r1Cluster, false), + topicDispatchRateForR1); + assertEquals(admin.topicPolicies().getReplicatorDispatchRate(topic, r1Cluster, true), + topicDispatchRateForR1); + }); + + // r1 cluster's dispatch rate is removed. + // If applied is true, return default dispatch rate, otherwise, return null. + admin.topicPolicies().removeReplicatorDispatchRate(topic, r1Cluster); + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topicPolicies().getReplicatorDispatchRate(topic, r1Cluster, true), defaultDispatchRateOnTopic); + assertNull(admin.topicPolicies().getReplicatorDispatchRate(topic, r1Cluster, false)); + }); + + // The default dispatch rate is removed. + // If applied is true, return namespace level config, otherwise, return null. + admin.topicPolicies().removeReplicatorDispatchRate(topic); + Awaitility.await().untilAsserted(() -> { + assertNull(admin.topicPolicies().getReplicatorDispatchRate(topic, false)); + assertEquals(admin.topicPolicies().getReplicatorDispatchRate(topic, true), defaultDispatchRateOnNamespace); + }); + + // Remove the replicator dispatch rate for the r1 cluster. + admin.namespaces().removeReplicatorDispatchRate(namespace, r1Cluster); + assertEquals(admin.namespaces().getReplicatorDispatchRate(namespace), defaultDispatchRateOnNamespace); + assertNull(admin.namespaces().getReplicatorDispatchRate(namespace, r1Cluster)); + + // Remove the default replicator dispatch rate. + admin.namespaces().removeReplicatorDispatchRate(namespace); + assertNull(admin.namespaces().getReplicatorDispatchRate(namespace)); + + // The broker config is used when the replicator dispatch rate is not set on any level. + Awaitility.await().untilAsserted(() -> assertReplicatorDispatchRateByBrokerConfig( + admin.topicPolicies().getReplicatorDispatchRate(topic, r1Cluster, true))); + } + + private void assertReplicatorDispatchRateByBrokerConfig(DispatchRate replicatorDispatchRate) { + assertNotNull(replicatorDispatchRate); + assertEquals(replicatorDispatchRate.getDispatchThrottlingRateInByte(), + conf.getDispatchThrottlingRatePerReplicatorInByte()); + assertEquals(replicatorDispatchRate.getDispatchThrottlingRateInByte(), + conf.getDispatchThrottlingRatePerReplicatorInMsg()); + } +} \ No newline at end of file diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminResourceGroupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminResourceGroupTest.java new file mode 100644 index 0000000000000..c7ac08aabe6e0 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminResourceGroupTest.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter; +import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.DispatchRate; +import org.apache.pulsar.common.policies.data.ResourceGroup; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +/** + * Unit test {@link AdminResource}. + */ +@Test(groups = "broker-admin") +public class AdminResourceGroupTest extends BrokerTestBase { + + @BeforeClass + @Override + public void setup() throws Exception { + super.baseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + conf.setTopicLevelPoliciesEnabled(true); + conf.setSystemTopicEnabled(true); + } + + @Test + public void testTopicResourceGroup() throws PulsarAdminException { + String topic = newTopicName(); + TopicName topicName = TopicName.get(topic); + + String resourceGroupName = "rg-topic-" + UUID.randomUUID(); + ResourceGroup resourceGroup = new ResourceGroup(); + resourceGroup.setPublishRateInMsgs(1000); + resourceGroup.setPublishRateInBytes(100000L); + resourceGroup.setDispatchRateInMsgs(2000); + resourceGroup.setDispatchRateInBytes(200000L); + admin.resourcegroups().createResourceGroup(resourceGroupName, resourceGroup); + + admin.topics().createNonPartitionedTopic(topic); + + admin.topicPolicies().setResourceGroup(topic, resourceGroupName); + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topicPolicies().getResourceGroup(topic, true), resourceGroupName); + }); + + Awaitility.await().untilAsserted(() -> { + org.apache.pulsar.broker.resourcegroup.ResourceGroup rg = pulsar.getResourceGroupServiceManager() + .getTopicResourceGroup(topicName); + assertNotNull(rg); + assertEquals(rg.resourceGroupName, resourceGroupName); + }); + + assertThrows(PulsarAdminException.PreconditionFailedException.class, () -> { + admin.resourcegroups().deleteResourceGroup(resourceGroupName); + }); + + admin.topicPolicies().removeResourceGroup(topic); + Awaitility.await().untilAsserted(() -> { + assertTrue(StringUtils.isEmpty(admin.topicPolicies().getResourceGroup(topic, true))); + org.apache.pulsar.broker.resourcegroup.ResourceGroup rg = pulsar.getResourceGroupServiceManager() + .getTopicResourceGroup(topicName); + assertNull(rg); + }); + admin.resourcegroups().deleteResourceGroup(resourceGroupName); + } + + @Test + public void testTopicResourceGroupOverriderNamespaceResourceGroup() throws PulsarAdminException { + String namespaceResourceGroupName = "rg-ns-" + UUID.randomUUID(); + ResourceGroup namespaceResourceGroup = new ResourceGroup(); + namespaceResourceGroup.setPublishRateInMsgs(1001); + namespaceResourceGroup.setPublishRateInBytes(100001L); + namespaceResourceGroup.setDispatchRateInMsgs(2001); + namespaceResourceGroup.setDispatchRateInBytes(200001L); + admin.resourcegroups().createResourceGroup(namespaceResourceGroupName, namespaceResourceGroup); + + String topicResourceGroupName = "rg-topic-" + UUID.randomUUID(); + ResourceGroup topicResourceGroup = new ResourceGroup(); + topicResourceGroup.setPublishRateInMsgs(1000); + topicResourceGroup.setPublishRateInBytes(100000L); + topicResourceGroup.setDispatchRateInMsgs(2000); + topicResourceGroup.setDispatchRateInBytes(200000L); + admin.resourcegroups().createResourceGroup(topicResourceGroupName, topicResourceGroup); + + String topic = newTopicName(); + TopicName topicName = TopicName.get(topic); + String namespace = topicName.getNamespace(); + admin.namespaces().setNamespaceResourceGroup(namespace, namespaceResourceGroupName); + assertEquals(admin.namespaces().getNamespaceResourceGroup(namespace), namespaceResourceGroupName); + + admin.topics().createNonPartitionedTopic(topic); + admin.topicPolicies().setResourceGroup(topic, topicResourceGroupName); + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topicPolicies() + .getResourceGroup(topic, true), topicResourceGroupName); + org.apache.pulsar.broker.resourcegroup.ResourceGroup rg = pulsar.getResourceGroupServiceManager() + .getTopicResourceGroup(topicName); + assertNotNull(rg); + assertEquals(rg.resourceGroupName, topicResourceGroupName); + }); + + assertThrows(PulsarAdminException.PreconditionFailedException.class, () -> { + admin.resourcegroups().deleteResourceGroup(topicResourceGroupName); + }); + assertThrows(PulsarAdminException.PreconditionFailedException.class, () -> { + admin.resourcegroups().deleteResourceGroup(namespaceResourceGroupName); + }); + + admin.topicPolicies().removeResourceGroup(topic); + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topicPolicies() + .getResourceGroup(topic, true), namespaceResourceGroupName); + org.apache.pulsar.broker.resourcegroup.ResourceGroup rg = pulsar.getResourceGroupServiceManager() + .getTopicResourceGroup(topicName); + assertNull(rg); + }); + admin.resourcegroups().deleteResourceGroup(topicResourceGroupName); + admin.namespaces().removeNamespaceResourceGroup(namespace); + Awaitility.await().untilAsserted(() -> { + org.apache.pulsar.broker.resourcegroup.ResourceGroup rg = pulsar.getResourceGroupServiceManager() + .getNamespaceResourceGroup(topicName.getNamespaceObject()); + assertNull(rg); + }); + admin.resourcegroups().deleteResourceGroup(namespaceResourceGroupName); + } + + @Test + public void testUpdateResourceGroup() throws PulsarAdminException { + String resourceGroupName = "rg-" + UUID.randomUUID(); + ResourceGroup resourceGroup = new ResourceGroup(); + resourceGroup.setPublishRateInMsgs(1000); + resourceGroup.setPublishRateInBytes(100000L); + resourceGroup.setDispatchRateInMsgs(2000); + resourceGroup.setDispatchRateInBytes(200000L); + resourceGroup.setReplicationDispatchRateInMsgs(10L); + resourceGroup.setReplicationDispatchRateInBytes(20L); + + admin.resourcegroups().createResourceGroup(resourceGroupName, resourceGroup); + ResourceGroup got = admin.resourcegroups().getResourceGroup(resourceGroupName); + assertEquals(got, resourceGroup); + + resourceGroup.setReplicationDispatchRateInMsgs(11L); + resourceGroup.setReplicationDispatchRateInBytes(29L); + admin.resourcegroups().updateResourceGroup(resourceGroupName, resourceGroup); + got = admin.resourcegroups().getResourceGroup(resourceGroupName); + assertEquals(got, resourceGroup); + + admin.resourcegroups().deleteResourceGroup(resourceGroupName); + assertThrows(PulsarAdminException.NotFoundException.class, + () -> admin.resourcegroups().getResourceGroup(resourceGroupName)); + } + + @Test + public void testReplicatorDispatchRate() throws PulsarAdminException { + String resourceGroupName = "rg-" + UUID.randomUUID(); + ResourceGroup resourceGroup = new ResourceGroup(); + resourceGroup.setPublishRateInMsgs(1000); + resourceGroup.setPublishRateInBytes(100000L); + resourceGroup.setDispatchRateInMsgs(2000); + resourceGroup.setDispatchRateInBytes(200000L); + resourceGroup.setReplicationDispatchRateInMsgs(10L); + resourceGroup.setReplicationDispatchRateInBytes(20L); + + admin.resourcegroups().createResourceGroup(resourceGroupName, resourceGroup); + Awaitility.await().untilAsserted(() -> assertThat( + pulsar.getResourceGroupServiceManager().resourceGroupGet(resourceGroupName)).isNotNull()); + + org.apache.pulsar.broker.resourcegroup.ResourceGroup rgRef = + pulsar.getResourceGroupServiceManager().resourceGroupGet(resourceGroupName); + + AtomicReference r2Limiter = new AtomicReference<>(); + rgRef.registerReplicatorDispatchRateLimiter("r2", r2Limiter::set); + assertThat(r2Limiter.get()).isEqualTo(rgRef.getResourceGroupReplicationDispatchLimiter()); + + String targetCluster = "r2"; + DispatchRate dispatchRate = + DispatchRate.builder() + .dispatchThrottlingRateInByte(10) + .dispatchThrottlingRateInMsg(20) + .ratePeriodInSecond(100) + .relativeToPublishRate(false) + .build(); + admin.resourcegroups().setReplicatorDispatchRate(resourceGroupName, targetCluster, dispatchRate); + Awaitility.await().untilAsserted(() -> { + ResourceGroupDispatchLimiter resourceGroupDispatchLimiter = r2Limiter.get(); + assertThat(resourceGroupDispatchLimiter).satisfies(n -> { + assertThat(n.getDispatchRateOnByte()).isEqualTo(10); + assertThat(n.getDispatchRateOnMsg()).isEqualTo(20); + }); + }); + + assertThat(admin.resourcegroups().getReplicatorDispatchRate(resourceGroupName, targetCluster)) + .isEqualTo(dispatchRate); + admin.resourcegroups().removeReplicatorDispatchRate(resourceGroupName, targetCluster); + assertThat(admin.resourcegroups().getReplicatorDispatchRate(resourceGroupName, targetCluster)).isNull(); + + Awaitility.await().untilAsserted(() -> { + assertThat(r2Limiter.get()).isEqualTo(rgRef.getResourceGroupReplicationDispatchLimiter()); + }); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ResourceGroupsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ResourceGroupsTest.java index b8b4f0ea1601f..c78c742790dd1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ResourceGroupsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ResourceGroupsTest.java @@ -18,28 +18,29 @@ */ package org.apache.pulsar.broker.admin; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.spy; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.fail; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Set; -import org.apache.pulsar.broker.admin.v2.ResourceGroups; +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; -import org.apache.pulsar.broker.web.RestException; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.ResourceGroup; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.awaitility.Awaitility; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -public class ResourceGroupsTest extends MockedPulsarServiceBaseTest { - private ResourceGroups resourcegroups; +@Slf4j +public class ResourceGroupsTest extends MockedPulsarServiceBaseTest { private List expectedRgNames = new ArrayList<>(); private final String testCluster = "test"; private final String testTenant = "test-tenant"; @@ -50,14 +51,6 @@ public class ResourceGroupsTest extends MockedPulsarServiceBaseTest { @Override protected void setup() throws Exception { super.internalSetup(); - resourcegroups = spy(ResourceGroups.class); - resourcegroups.setServletContext(new MockServletContext()); - resourcegroups.setPulsar(pulsar); - doReturn(false).when(resourcegroups).isRequestHttps(); - doReturn("test").when(resourcegroups).clientAppId(); - doReturn(null).when(resourcegroups).originalPrincipal(); - doReturn(null).when(resourcegroups).clientAuthData(); - prepareData(); } @@ -70,16 +63,13 @@ protected void cleanup() throws Exception { @Test public void testCrudResourceGroups() throws Exception { // create with null resourcegroup should fail. - try { - resourcegroups.createOrUpdateResourceGroup("test-resourcegroup-invalid", null); - fail("should have failed"); - } catch (RestException e) { - //Ok. - } + assertThatThrownBy(() -> { + admin.resourcegroups().createResourceGroup("test-resourcegroup-invalid", null); + }).isInstanceOf(PulsarAdminException.class); // create resourcegroup with default values ResourceGroup testResourceGroupOne = new ResourceGroup(); - resourcegroups.createOrUpdateResourceGroup("test-resourcegroup-one", testResourceGroupOne); + admin.resourcegroups().createResourceGroup("test-resourcegroup-one", testResourceGroupOne); expectedRgNames.add("test-resourcegroup-one"); // create resourcegroup with non default values. @@ -89,16 +79,13 @@ public void testCrudResourceGroups() throws Exception { testResourceGroupTwo.setPublishRateInMsgs(100); testResourceGroupTwo.setPublishRateInBytes(10000L); - resourcegroups.createOrUpdateResourceGroup("test-resourcegroup-two", testResourceGroupTwo); + admin.resourcegroups().createResourceGroup("test-resourcegroup-two", testResourceGroupTwo); expectedRgNames.add("test-resourcegroup-two"); // null resourcegroup update should fail. - try { - resourcegroups.createOrUpdateResourceGroup("test-resourcegroup-one", null); - fail("should have failed"); - } catch (RestException e) { - //Ok. - } + assertThatThrownBy(() -> { + admin.resourcegroups().createResourceGroup("test-resourcegroup-one", null); + }).isInstanceOf(PulsarAdminException.class); // update with some real values ResourceGroup testResourceGroupOneUpdate = new ResourceGroup(); @@ -106,35 +93,29 @@ public void testCrudResourceGroups() throws Exception { testResourceGroupOneUpdate.setDispatchRateInBytes(5000L); testResourceGroupOneUpdate.setPublishRateInMsgs(10); testResourceGroupOneUpdate.setPublishRateInBytes(1000L); - resourcegroups.createOrUpdateResourceGroup("test-resourcegroup-one", testResourceGroupOneUpdate); + admin.resourcegroups().createResourceGroup("test-resourcegroup-one", testResourceGroupOneUpdate); // get a non existent resourcegroup - try { - resourcegroups.getResourceGroup("test-resourcegroup-invalid"); - fail("should have failed"); - } catch (RestException e) { - //Ok - } + assertThatThrownBy(() -> { + admin.resourcegroups().getResourceGroup("test-resourcegroup-invalid"); + }).isInstanceOf(PulsarAdminException.class); // get list of all resourcegroups - List gotRgNames = resourcegroups.getResourceGroups(); + List gotRgNames = admin.resourcegroups().getResourceGroups(); assertEquals(gotRgNames.size(), expectedRgNames.size()); Collections.sort(gotRgNames); Collections.sort(expectedRgNames); assertEquals(gotRgNames, expectedRgNames); // delete a non existent resourcegroup - try { - resourcegroups.deleteResourceGroup("test-resourcegroup-invalid"); - fail("should have failed"); - } catch (RestException e) { - //Ok - } + assertThatThrownBy(() -> { + admin.resourcegroups().getResourceGroup("test-resourcegroup-invalid"); + }).isInstanceOf(PulsarAdminException.class); // delete the ResourceGroups we created. - Iterator rgIterator = expectedRgNames.iterator(); - while (rgIterator.hasNext()) { - resourcegroups.deleteResourceGroup(rgIterator.next()); + Iterator rg_Iterator = expectedRgNames.iterator(); + while (rg_Iterator.hasNext()) { + admin.resourcegroups().deleteResourceGroup(rg_Iterator.next()); } } @@ -147,27 +128,28 @@ public void testNamespaceResourceGroup() throws Exception { testResourceGroupTwo.setPublishRateInMsgs(100); testResourceGroupTwo.setPublishRateInBytes(10000L); - resourcegroups.createOrUpdateResourceGroup("test-resourcegroup-three", testResourceGroupTwo); + admin.resourcegroups().createResourceGroup("test-resourcegroup-three", testResourceGroupTwo); admin.namespaces().createNamespace(testNameSpace); // set invalid ResourceGroup in namespace - try { + assertThatThrownBy(() -> { admin.namespaces().setNamespaceResourceGroup(testNameSpace, "test-resourcegroup-invalid"); - fail("should have failed"); - } catch (Exception e) { - //Ok. - } + }).isInstanceOf(PulsarAdminException.class); + // set resourcegroup in namespace admin.namespaces().setNamespaceResourceGroup(testNameSpace, "test-resourcegroup-three"); + Awaitility.await().untilAsserted(() -> assertNotNull(pulsar.getResourceGroupServiceManager() + .getNamespaceResourceGroup(NamespaceName.get(testNameSpace)))); // try deleting the resourcegroup, should fail - try { - resourcegroups.deleteResourceGroup("test-resourcegroup-three"); - } catch (RestException e) { - //Ok - } + assertThatThrownBy(() -> { + admin.resourcegroups().deleteResourceGroup("test-resourcegroup-three"); + }).isInstanceOf(PulsarAdminException.class); + // remove resourcegroup from namespace admin.namespaces().removeNamespaceResourceGroup(testNameSpace); - resourcegroups.deleteResourceGroup("test-resourcegroup-three"); + Awaitility.await().untilAsserted(() -> assertNull(pulsar.getResourceGroupServiceManager() + .getNamespaceResourceGroup(NamespaceName.get(testNameSpace)))); + admin.resourcegroups().deleteResourceGroup("test-resourcegroup-three"); } private void prepareData() throws PulsarAdminException { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/RGUsageMTAggrWaitForAllMsgsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/RGUsageMTAggrWaitForAllMsgsTest.java index 7d78cabc29b97..6d74ef74528d5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/RGUsageMTAggrWaitForAllMsgsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/RGUsageMTAggrWaitForAllMsgsTest.java @@ -596,13 +596,13 @@ private void verifyRGProdConsStats(String[] topicStrings, final String tenantRGName = topicToTenantRGName(topic); if (!rgsWithPublishStatsGathered.contains(tenantRGName)) { prodCounts = this.rgservice.getRGUsage(tenantRGName, ResourceGroupMonitoringClass.Publish, - getCumulativeUsageStats); + getCumulativeUsageStats).entrySet().iterator().next().getValue(); totalTenantRGProdCounts = ResourceGroup.accumulateBMCount(totalTenantRGProdCounts, prodCounts); rgsWithPublishStatsGathered.add(tenantRGName); } if (!rgsWithDispatchStatsGathered.contains(tenantRGName)) { consCounts = this.rgservice.getRGUsage(tenantRGName, ResourceGroupMonitoringClass.Dispatch, - getCumulativeUsageStats); + getCumulativeUsageStats).entrySet().iterator().next().getValue(); totalTenantRGConsCounts = ResourceGroup.accumulateBMCount(totalTenantRGConsCounts, consCounts); rgsWithDispatchStatsGathered.add(tenantRGName); } @@ -613,13 +613,13 @@ private void verifyRGProdConsStats(String[] topicStrings, if (tenantRGName.compareTo(nsRGName) != 0) { if (!rgsWithPublishStatsGathered.contains(nsRGName)) { prodCounts = this.rgservice.getRGUsage(nsRGName, ResourceGroupMonitoringClass.Publish, - getCumulativeUsageStats); + getCumulativeUsageStats).entrySet().iterator().next().getValue(); totalNsRGProdCounts = ResourceGroup.accumulateBMCount(totalNsRGProdCounts, prodCounts); rgsWithPublishStatsGathered.add(nsRGName); } if (!rgsWithDispatchStatsGathered.contains(nsRGName)) { consCounts = this.rgservice.getRGUsage(nsRGName, ResourceGroupMonitoringClass.Dispatch, - getCumulativeUsageStats); + getCumulativeUsageStats).entrySet().iterator().next().getValue(); totalNsRGConsCounts = ResourceGroup.accumulateBMCount(totalNsRGConsCounts, consCounts); rgsWithDispatchStatsGathered.add(nsRGName); } @@ -683,11 +683,17 @@ private void verifyRGMetrics(int sentNumBytes, int sentNumMsgs, for (ResourceGroupMonitoringClass mc : ResourceGroupMonitoringClass.values()) { String mcName = mc.name(); int mcIndex = mc.ordinal(); - totalQuotaBytes[mcIndex] += ResourceGroupService.getRgQuotaByteCount(rgName, mcName); - totalQuotaMessages[mcIndex] += ResourceGroupService.getRgQuotaMessageCount(rgName, mcName); - totalUsedBytes[mcIndex] += ResourceGroupService.getRgLocalUsageByteCount(rgName, mcName); - totalUsedMessages[mcIndex] += ResourceGroupService.getRgLocalUsageMessageCount(rgName, mcName); - totalUsageReportCounts[mcIndex] += ResourceGroup.getRgUsageReportedCount(rgName, mcName); + double quotaBytes = ResourceGroupService.getRgQuotaByteCount(rgName, mcName, pulsar.getConfiguration().getClusterName(),null); + totalQuotaBytes[mcIndex] += quotaBytes; + double quotaMesgs = ResourceGroupService.getRgQuotaMessageCount(rgName, mcName,pulsar.getConfiguration().getClusterName(),null); + totalQuotaMessages[mcIndex] += quotaMesgs; + double usedBytes = ResourceGroupService.getRgLocalUsageByteCount(rgName, mcName,pulsar.getConfiguration().getClusterName(),null); + totalUsedBytes[mcIndex] += usedBytes; + double usedMesgs = ResourceGroupService.getRgLocalUsageMessageCount(rgName, mcName,pulsar.getConfiguration().getClusterName(),null); + totalUsedMessages[mcIndex] += usedMesgs; + + double usageReportedCount = ResourceGroup.getRgUsageReportedCount(rgName, mcName); + totalUsageReportCounts[mcIndex] += usageReportedCount; } totalTenantRegisters += ResourceGroupService.getRgTenantRegistersCount(rgName); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupMetricTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupMetricTest.java new file mode 100644 index 0000000000000..bffb17f6d35cf --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupMetricTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.resourcegroup; + +import static org.testng.Assert.assertEquals; +import org.apache.pulsar.broker.resourcegroup.ResourceGroup.BytesAndMessagesCount; +import org.apache.pulsar.broker.resourcegroup.ResourceGroup.ResourceGroupMonitoringClass; +import org.testng.annotations.Test; + +public class ResourceGroupMetricTest { + @Test + public void testLocalQuotaMetric() { + String rgName = "my-resource-group"; + ResourceGroupMonitoringClass publish = ResourceGroupMonitoringClass.Publish; + int reportPeriod = 2; + BytesAndMessagesCount b = new BytesAndMessagesCount(); + b.messages = 10; + b.bytes = 20; + int incTimes = 2; + for (int i = 0; i < 2; i++) { + ResourceGroupService.incRgCalculatedQuota(rgName, publish, b, reportPeriod, "local","remote"); + } + double rgLocalUsageByteCount = ResourceGroupService.getRgQuotaByteCount(rgName, publish.name(),"local", "remote"); + double rgQuotaMessageCount = ResourceGroupService.getRgQuotaMessageCount(rgName, publish.name(),"local", "remote"); + assertEquals(rgLocalUsageByteCount, incTimes * b.bytes * reportPeriod); + assertEquals(rgQuotaMessageCount, incTimes * b.messages * reportPeriod); + + double rgLocalUsageByte = ResourceGroupService.getRgQuotaByte(rgName, publish.name(),"local", "remote"); + double rgQuotaMessage = ResourceGroupService.getRgQuotaMessage(rgName, publish.name(),"local", "remote"); + assertEquals(rgLocalUsageByte, b.bytes); + assertEquals(rgQuotaMessage, b.messages); + } +} \ No newline at end of file diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManagerTest.java new file mode 100644 index 0000000000000..3b13f04e66f2b --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManagerTest.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.resourcegroup; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import java.util.concurrent.TimeUnit; +import org.apache.pulsar.broker.resourcegroup.ResourceGroup.BytesAndMessagesCount; +import org.awaitility.Awaitility; +import org.testng.annotations.Test; + +public class ResourceGroupRateLimiterManagerTest { + + @Test + public void testNewReplicationDispatchRateLimiterWithEmptyResourceGroup() { + org.apache.pulsar.common.policies.data.ResourceGroup emptyResourceGroup = + new org.apache.pulsar.common.policies.data.ResourceGroup(); + + ResourceGroupDispatchLimiter resourceGroupDispatchLimiter = + ResourceGroupRateLimiterManager.newReplicationDispatchRateLimiter(emptyResourceGroup); + assertFalse(resourceGroupDispatchLimiter.isDispatchRateLimitingEnabled()); + + assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnMsg(), -1L); + assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnMsg(), -1L); + + assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnByte(), -1L); + assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnByte(), -1L); + } + + @Test + public void testReplicationDispatchRateLimiterOnMsgs() { + org.apache.pulsar.common.policies.data.ResourceGroup resourceGroup = + new org.apache.pulsar.common.policies.data.ResourceGroup(); + resourceGroup.setReplicationDispatchRateInMsgs(10L); + ResourceGroupDispatchLimiter resourceGroupDispatchLimiter = + ResourceGroupRateLimiterManager.newReplicationDispatchRateLimiter(resourceGroup); + assertTrue(resourceGroupDispatchLimiter.isDispatchRateLimitingEnabled()); + + + assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnMsg(), resourceGroup.getReplicationDispatchRateInMsgs().longValue()); + assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnMsg(), resourceGroup.getReplicationDispatchRateInMsgs().longValue()); + + assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnByte(), -1L); + assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnByte(), -1L); + } + + @Test + public void testReplicationDispatchRateLimiterOnBytes() { + org.apache.pulsar.common.policies.data.ResourceGroup resourceGroup = + new org.apache.pulsar.common.policies.data.ResourceGroup(); + resourceGroup.setReplicationDispatchRateInBytes(20L); + ResourceGroupDispatchLimiter resourceGroupDispatchLimiter = + ResourceGroupRateLimiterManager.newReplicationDispatchRateLimiter(resourceGroup); + assertTrue(resourceGroupDispatchLimiter.isDispatchRateLimitingEnabled()); + + assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnMsg(), -1L); + assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnMsg(), -1L); + + assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnByte(), resourceGroup.getReplicationDispatchRateInBytes().longValue()); + assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnByte(), resourceGroup.getReplicationDispatchRateInBytes().longValue()); + } + + @Test + public void testUpdateReplicationDispatchRateLimiter() { + org.apache.pulsar.common.policies.data.ResourceGroup resourceGroup = + new org.apache.pulsar.common.policies.data.ResourceGroup(); + resourceGroup.setReplicationDispatchRateInMsgs(10L); + resourceGroup.setReplicationDispatchRateInBytes(100L); + + ResourceGroupDispatchLimiter resourceGroupDispatchLimiter = + ResourceGroupRateLimiterManager.newReplicationDispatchRateLimiter(resourceGroup); + + BytesAndMessagesCount quota = new BytesAndMessagesCount(); + quota.messages = 20; + quota.bytes = 200; + ResourceGroupRateLimiterManager.updateReplicationDispatchRateLimiter(resourceGroupDispatchLimiter, quota); + + assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnByte(), quota.bytes); + assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnByte(), quota.bytes); + assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnMsg(), quota.messages); + assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnMsg(), quota.messages); + } + + @Test + public void newReplicationDispatchRateLimiterWithValidData() { + ResourceGroupDispatchLimiter limiter = + ResourceGroupRateLimiterManager.newReplicationDispatchRateLimiter(100L, 1000L); + assertEquals(limiter.getDispatchRateOnMsg(), 100L); + assertEquals(limiter.getDispatchRateOnByte(), 1000L); + } + + @Test + public void newReplicationDispatchRateLimiterWithZeroValues() { + ResourceGroupDispatchLimiter limiter = + ResourceGroupRateLimiterManager.newReplicationDispatchRateLimiter(0L, 0L); + assertEquals(limiter.getDispatchRateOnMsg(), -1L); + assertEquals(limiter.getDispatchRateOnByte(), -1L); + } + + @Test + public void newReplicationDispatchRateLimiterWithNegativeValues() { + ResourceGroupDispatchLimiter limiter = + ResourceGroupRateLimiterManager.newReplicationDispatchRateLimiter(-1L, -1L); + assertEquals(limiter.getDispatchRateOnMsg(), -1L); + assertEquals(limiter.getDispatchRateOnByte(), -1L); + } + + @Test + public void testConsumeDispatchQuotaWithAsyncTokenBucket() { + ResourceGroupDispatchLimiter limiter = + ResourceGroupRateLimiterManager.newReplicationDispatchRateLimiter(5L, 50L); + + limiter.consumeDispatchQuota(2, 20); + + assertEquals(limiter.getAvailableDispatchRateLimitOnMsg(), 3L); + assertEquals(limiter.getAvailableDispatchRateLimitOnByte(), 30L); + } + + @Test + public void testLimiterRefillsAfterRatePeriod() { + ResourceGroupDispatchLimiter limiter = + ResourceGroupRateLimiterManager.newReplicationDispatchRateLimiter(2L, -1L); + + limiter.consumeDispatchQuota(2, 0); + assertEquals(limiter.getAvailableDispatchRateLimitOnMsg(), 0L); + + Awaitility.await().atMost(2, TimeUnit.SECONDS) + .untilAsserted(() -> assertEquals(limiter.getAvailableDispatchRateLimitOnMsg(), 2L)); + } + + @Test + public void testCloseDisablesLimiter() { + ResourceGroupDispatchLimiter limiter = + ResourceGroupRateLimiterManager.newReplicationDispatchRateLimiter(5L, 50L); + + limiter.close(); + + assertFalse(limiter.isDispatchRateLimitingEnabled()); + assertEquals(limiter.getDispatchRateOnMsg(), -1L); + assertEquals(limiter.getDispatchRateOnByte(), -1L); + assertEquals(limiter.getAvailableDispatchRateLimitOnMsg(), -1L); + assertEquals(limiter.getAvailableDispatchRateLimitOnByte(), -1L); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterTest.java index f06fc638bc888..51ccdeaa1a6a9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterTest.java @@ -130,6 +130,11 @@ private void testRateLimit() throws PulsarAdminException, PulsarClientException, // Now detach the namespace admin.namespaces().removeNamespaceResourceGroup(namespaceName); + Awaitility.await().untilAsserted(() -> { + ResourceGroup namespaceResourceGroup = pulsar.getResourceGroupServiceManager() + .getNamespaceResourceGroup(NamespaceName.get(namespaceName)); + assertNull(namespaceResourceGroup); + }); deleteResourceGroup(rgName); Thread.sleep(2000); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupReportLocalUsageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupReportLocalUsageTest.java index 139d19886c7d1..3940db2b9f816 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupReportLocalUsageTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupReportLocalUsageTest.java @@ -72,14 +72,28 @@ public long computeLocalQuota(long confUsage, long myUsage, long[] allUsages) { rgConfig.setPublishRateInMsgs(2000); service.resourceGroupCreate(rgName, rgConfig); - BytesAndMessagesCount bytesAndMessagesCount = new BytesAndMessagesCount(); - bytesAndMessagesCount.bytes = 20; - bytesAndMessagesCount.messages = 10; + BytesAndMessagesCount dispatchBM = new BytesAndMessagesCount(); + dispatchBM.bytes = 20; + dispatchBM.messages = 10; + BytesAndMessagesCount publishBM = new BytesAndMessagesCount(); + publishBM.bytes = 30; + publishBM.messages = 20; + String replicator1RemoteCluster = "r1"; + BytesAndMessagesCount replicator1BM = new BytesAndMessagesCount(); + replicator1BM.bytes = 40; + replicator1BM.messages = 30; + String replicator2RemoteCluster = "r2"; + BytesAndMessagesCount replicator2BM = new BytesAndMessagesCount(); + replicator2BM.bytes = 50; + replicator2BM.messages = 40; org.apache.pulsar.broker.resourcegroup.ResourceGroup resourceGroup = service.resourceGroupGet(rgName); - for (ResourceGroupMonitoringClass value : ResourceGroupMonitoringClass.values()) { - resourceGroup.incrementLocalUsageStats(value, bytesAndMessagesCount); - } + resourceGroup.incrementLocalUsageStats(ResourceGroupMonitoringClass.Dispatch, dispatchBM, null); + resourceGroup.incrementLocalUsageStats(ResourceGroupMonitoringClass.Publish, publishBM, null); + resourceGroup.incrementLocalUsageStats(ResourceGroupMonitoringClass.ReplicationDispatch, replicator1BM, + replicator1RemoteCluster); + resourceGroup.incrementLocalUsageStats(ResourceGroupMonitoringClass.ReplicationDispatch, replicator2BM, + replicator2RemoteCluster); // Case1: Suppress report ResourceUsage. needReport.set(false); @@ -87,35 +101,75 @@ public long computeLocalQuota(long confUsage, long myUsage, long[] allUsages) { resourceGroup.rgFillResourceUsage(resourceUsage); assertFalse(resourceUsage.hasDispatch()); assertFalse(resourceUsage.hasPublish()); - for (ResourceGroupMonitoringClass value : ResourceGroupMonitoringClass.values()) { - PerMonitoringClassFields monitoredEntity = - resourceGroup.getMonitoredEntity(value); - assertEquals(monitoredEntity.usedLocallySinceLastReport.messages, 0); - assertEquals(monitoredEntity.usedLocallySinceLastReport.bytes, 0); - assertEquals(monitoredEntity.totalUsedLocally.messages, 0); - assertEquals(monitoredEntity.totalUsedLocally.bytes, 0); - assertEquals(monitoredEntity.lastReportedValues.messages, 0); - assertEquals(monitoredEntity.lastReportedValues.bytes, 0); - } + assertEquals(resourceUsage.getReplicatorsCount(), 0); + + PerMonitoringClassFields dispatchMonitoredEntity = + resourceGroup.getMonitoredEntity(ResourceGroupMonitoringClass.Dispatch, null); + assertEquals(dispatchMonitoredEntity.usedLocallySinceLastReport.messages, 0); + assertEquals(dispatchMonitoredEntity.usedLocallySinceLastReport.bytes, 0); + assertEquals(dispatchMonitoredEntity.lastReportedValues.messages, 0); + assertEquals(dispatchMonitoredEntity.lastReportedValues.bytes, 0); + PerMonitoringClassFields publishMonitoredEntity = + resourceGroup.getMonitoredEntity(ResourceGroupMonitoringClass.Publish, null); + assertEquals(publishMonitoredEntity.usedLocallySinceLastReport.messages, 0); + assertEquals(publishMonitoredEntity.usedLocallySinceLastReport.bytes, 0); + assertEquals(publishMonitoredEntity.lastReportedValues.messages, 0); + assertEquals(publishMonitoredEntity.lastReportedValues.bytes, 0); + PerMonitoringClassFields r1MonitoredEntity = + resourceGroup.getMonitoredEntity(ResourceGroupMonitoringClass.ReplicationDispatch, + replicator1RemoteCluster); + assertEquals(r1MonitoredEntity.usedLocallySinceLastReport.messages, 0); + assertEquals(r1MonitoredEntity.usedLocallySinceLastReport.bytes, 0); + assertEquals(r1MonitoredEntity.lastReportedValues.messages, 0); + assertEquals(r1MonitoredEntity.lastReportedValues.bytes, 0); + PerMonitoringClassFields r2MonitoredEntity = + resourceGroup.getMonitoredEntity(ResourceGroupMonitoringClass.ReplicationDispatch, + replicator2RemoteCluster); + assertEquals(r2MonitoredEntity.usedLocallySinceLastReport.messages, 0); + assertEquals(r2MonitoredEntity.usedLocallySinceLastReport.bytes, 0); + assertEquals(r2MonitoredEntity.lastReportedValues.messages, 0); + assertEquals(r2MonitoredEntity.lastReportedValues.bytes, 0); // Case2: Report ResourceUsage. - for (ResourceGroupMonitoringClass value : ResourceGroupMonitoringClass.values()) { - resourceGroup.incrementLocalUsageStats(value, bytesAndMessagesCount); - } + resourceGroup.incrementLocalUsageStats(ResourceGroupMonitoringClass.Dispatch, dispatchBM, null); + resourceGroup.incrementLocalUsageStats(ResourceGroupMonitoringClass.Publish, publishBM, null); + resourceGroup.incrementLocalUsageStats(ResourceGroupMonitoringClass.ReplicationDispatch, replicator1BM, + replicator1RemoteCluster); + resourceGroup.incrementLocalUsageStats(ResourceGroupMonitoringClass.ReplicationDispatch, replicator2BM, + replicator2RemoteCluster); needReport.set(true); + resourceUsage = new ResourceUsage(); resourceGroup.rgFillResourceUsage(resourceUsage); assertTrue(resourceUsage.hasDispatch()); assertTrue(resourceUsage.hasPublish()); - for (ResourceGroupMonitoringClass value : ResourceGroupMonitoringClass.values()) { - PerMonitoringClassFields monitoredEntity = - resourceGroup.getMonitoredEntity(value); - assertEquals(monitoredEntity.usedLocallySinceLastReport.messages, 0); - assertEquals(monitoredEntity.usedLocallySinceLastReport.bytes, 0); - assertEquals(monitoredEntity.totalUsedLocally.messages, bytesAndMessagesCount.messages); - assertEquals(monitoredEntity.totalUsedLocally.bytes, bytesAndMessagesCount.bytes); - assertEquals(monitoredEntity.lastReportedValues.messages, bytesAndMessagesCount.messages); - assertEquals(monitoredEntity.lastReportedValues.bytes, bytesAndMessagesCount.bytes); - } + assertEquals(resourceUsage.getReplicatorsCount(), 2); + + dispatchMonitoredEntity = + resourceGroup.getMonitoredEntity(ResourceGroupMonitoringClass.Dispatch, null); + assertEquals(dispatchMonitoredEntity.usedLocallySinceLastReport.messages, 0); + assertEquals(dispatchMonitoredEntity.usedLocallySinceLastReport.bytes, 0); + assertEquals(dispatchMonitoredEntity.lastReportedValues.messages, dispatchBM.messages); + assertEquals(dispatchMonitoredEntity.lastReportedValues.bytes, dispatchBM.bytes); + publishMonitoredEntity = + resourceGroup.getMonitoredEntity(ResourceGroupMonitoringClass.Publish, null); + assertEquals(publishMonitoredEntity.usedLocallySinceLastReport.messages, 0); + assertEquals(publishMonitoredEntity.usedLocallySinceLastReport.bytes, 0); + assertEquals(publishMonitoredEntity.lastReportedValues.messages, publishBM.messages); + assertEquals(publishMonitoredEntity.lastReportedValues.bytes, publishBM.bytes); + r1MonitoredEntity = + resourceGroup.getMonitoredEntity(ResourceGroupMonitoringClass.ReplicationDispatch, + replicator1RemoteCluster); + assertEquals(r1MonitoredEntity.usedLocallySinceLastReport.messages, 0); + assertEquals(r1MonitoredEntity.usedLocallySinceLastReport.bytes, 0); + assertEquals(r1MonitoredEntity.lastReportedValues.messages, replicator1BM.messages); + assertEquals(r1MonitoredEntity.lastReportedValues.bytes, replicator1BM.bytes); + r2MonitoredEntity = + resourceGroup.getMonitoredEntity(ResourceGroupMonitoringClass.ReplicationDispatch, + replicator2RemoteCluster); + assertEquals(r2MonitoredEntity.usedLocallySinceLastReport.messages, 0); + assertEquals(r2MonitoredEntity.usedLocallySinceLastReport.bytes, 0); + assertEquals(r2MonitoredEntity.lastReportedValues.messages, replicator2BM.messages); + assertEquals(r2MonitoredEntity.lastReportedValues.bytes, replicator2BM.bytes); } } \ No newline at end of file diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java index 534492894cfba..ea618249e0c39 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java @@ -18,19 +18,33 @@ */ package org.apache.pulsar.broker.resourcegroup; -import java.util.HashSet; +import static org.mockito.Mockito.mock; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Policy.Expiration; +import com.google.common.collect.Lists; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.resourcegroup.ResourceGroup.BytesAndMessagesCount; +import org.apache.pulsar.broker.resourcegroup.ResourceGroup.PerBrokerUsageStats; import org.apache.pulsar.broker.resourcegroup.ResourceGroup.PerMonitoringClassFields; import org.apache.pulsar.broker.resourcegroup.ResourceGroup.ResourceGroupMonitoringClass; -import org.apache.pulsar.broker.service.resource.usage.NetworkUsage; +import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter; import org.apache.pulsar.broker.service.resource.usage.ResourceUsage; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.DispatchRate; +import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -49,8 +63,9 @@ protected void setup() throws Exception { @Override public boolean needToReportLocalUsage(long currentBytesUsed, long lastReportedBytes, long currentMessagesUsed, long lastReportedMessages, - long lastReportTimeMSecsSinceEpoch) { - final int maxSuppressRounds = ResourceGroupService.MaxUsageReportSuppressRounds; + long lastReportTimeMSecsSinceEpoch) + { + final int maxSuppressRounds = conf.getResourceUsageMaxUsageReportSuppressRounds(); if (++numLocalReportsEvaluated % maxSuppressRounds == (maxSuppressRounds - 1)) { return true; } @@ -86,7 +101,7 @@ protected void cleanup() throws Exception { @Test public void measureOpsTime() throws PulsarAdminException { long mSecsStart, mSecsEnd, diffMsecs; - final int numPerfTestIterations = 10_000_000; + final int numPerfTestIterations = 1_000; org.apache.pulsar.common.policies.data.ResourceGroup rgConfig = new org.apache.pulsar.common.policies.data.ResourceGroup(); BytesAndMessagesCount stats = new BytesAndMessagesCount(); @@ -101,7 +116,8 @@ public void measureOpsTime() throws PulsarAdminException { for (int ix = 0; ix < numPerfTestIterations; ix++) { for (int monClassIdx = 0; monClassIdx < ResourceGroupMonitoringClass.values().length; monClassIdx++) { monClass = ResourceGroupMonitoringClass.values()[monClassIdx]; - rg.incrementLocalUsageStats(monClass, stats); + rg.incrementLocalUsageStats(monClass, stats, + monClass.equals(ResourceGroupMonitoringClass.ReplicationDispatch) ? "r2" : null); } } mSecsEnd = System.currentTimeMillis(); @@ -110,16 +126,18 @@ public void measureOpsTime() throws PulsarAdminException { numPerfTestIterations, diffMsecs, (1000 * (float) diffMsecs) / numPerfTestIterations); // Going through the resource-group service - final String tenantName = "SomeTenant"; - final String namespaceName = "SomeNameSpace"; + final TopicName topicName = TopicName.get("SomeTenant/SomeNameSpace/my-topic"); + rgs.registerTopic(rgName, topicName); + final String tenantName = topicName.getTenant(); + final String namespaceName = topicName.getNamespace(); rgs.registerTenant(rgName, tenantName); - final NamespaceName tenantAndNamespaceName = NamespaceName.get(tenantName, namespaceName); + final NamespaceName tenantAndNamespaceName = topicName.getNamespaceObject(); rgs.registerNameSpace(rgName, tenantAndNamespaceName); mSecsStart = System.currentTimeMillis(); for (int ix = 0; ix < numPerfTestIterations; ix++) { for (int monClassIdx = 0; monClassIdx < ResourceGroupMonitoringClass.values().length; monClassIdx++) { monClass = ResourceGroupMonitoringClass.values()[monClassIdx]; - rgs.incrementUsage(tenantName, namespaceName, monClass, stats); + rgs.incrementUsage(tenantName, namespaceName, topicName.toString(), monClass, stats, monClass.equals(ResourceGroupMonitoringClass.ReplicationDispatch) ? "r2" : null); } } mSecsEnd = System.currentTimeMillis(); @@ -128,6 +146,7 @@ public void measureOpsTime() throws PulsarAdminException { numPerfTestIterations, diffMsecs, (1000 * (float) diffMsecs) / numPerfTestIterations); rgs.unRegisterTenant(rgName, tenantName); rgs.unRegisterNameSpace(rgName, tenantAndNamespaceName); + rgs.unRegisterTopic(topicName); // The overhead of a RG lookup mSecsStart = System.currentTimeMillis(); @@ -143,6 +162,129 @@ public void measureOpsTime() throws PulsarAdminException { } @Test + public void testReplicatorResourceGroupOps() throws PulsarAdminException { + String rgName = "testRG-" + System.currentTimeMillis(); + org.apache.pulsar.common.policies.data.ResourceGroup rgConfig = + new org.apache.pulsar.common.policies.data.ResourceGroup(); + rgConfig.setReplicationDispatchRateInBytes(2000L); + rgConfig.setReplicationDispatchRateInMsgs(400L); + + rgs.resourceGroupCreate(rgName, rgConfig); + ResourceGroup retRG = rgs.resourceGroupGet(rgName); + Assert.assertNotEquals(retRG, null); + + PerMonitoringClassFields r1 = retRG.getMonitoredEntity(ResourceGroupMonitoringClass.ReplicationDispatch, "r1"); + Assert.assertEquals(r1.configValuesPerPeriod.bytes, + rgConfig.getReplicationDispatchRateInBytes().longValue()); + Assert.assertEquals(r1.configValuesPerPeriod.messages, + rgConfig.getReplicationDispatchRateInMsgs().intValue()); + + AtomicReference r1Limiter = new AtomicReference<>(); + retRG.registerReplicatorDispatchRateLimiter("r1", r1Limiter::set); + AtomicReference r2Limiter = new AtomicReference<>(); + retRG.registerReplicatorDispatchRateLimiter("r2", r2Limiter::set); + + Assert.assertEquals(r1Limiter.get(), r2Limiter.get()); + Assert.assertEquals(r1Limiter.get().getDispatchRateOnByte(), + rgConfig.getReplicationDispatchRateInBytes().longValue()); + Assert.assertEquals(r1Limiter.get().getDispatchRateOnMsg(), + rgConfig.getReplicationDispatchRateInMsgs().longValue()); + + // Update the default rate limitation. + rgConfig.setReplicationDispatchRateInBytes(rgConfig.getReplicationDispatchRateInBytes() * 10); + rgConfig.setReplicationDispatchRateInMsgs(rgConfig.getReplicationDispatchRateInMsgs() * 10); + rgs.resourceGroupUpdate(rgName, rgConfig); + + Assert.assertEquals(r1Limiter.get(), r2Limiter.get()); + Assert.assertEquals(r1Limiter.get().getDispatchRateOnByte(), + rgConfig.getReplicationDispatchRateInBytes().longValue()); + Assert.assertEquals(r1Limiter.get().getDispatchRateOnMsg(), + rgConfig.getReplicationDispatchRateInMsgs().longValue()); + + r1 = retRG.getMonitoredEntity(ResourceGroupMonitoringClass.ReplicationDispatch, "r1"); + Assert.assertEquals(r1.configValuesPerPeriod.bytes, + rgConfig.getReplicationDispatchRateInBytes().longValue()); + Assert.assertEquals(r1.configValuesPerPeriod.messages, + rgConfig.getReplicationDispatchRateInMsgs().intValue()); + + // Set up the specific rate limitation based on the remote cluster. + Map replicatorLimiterMap = new ConcurrentHashMap<>(); + // r1 + DispatchRate newR1Limiter = + DispatchRate.builder().dispatchThrottlingRateInMsg(1024).dispatchThrottlingRateInByte(200).build(); + replicatorLimiterMap.put(retRG.getReplicatorDispatchRateLimiterKey("r1"), newR1Limiter); + rgConfig.setReplicatorDispatchRate(replicatorLimiterMap); + rgs.resourceGroupUpdate(rgName, rgConfig); + + Assert.assertNotEquals(r1Limiter.get(), r2Limiter.get()); + Assert.assertEquals(r1Limiter.get().getDispatchRateOnByte(), newR1Limiter.getDispatchThrottlingRateInByte()); + Assert.assertEquals(r1Limiter.get().getDispatchRateOnMsg(), newR1Limiter.getDispatchThrottlingRateInMsg()); + Assert.assertEquals(r2Limiter.get().getDispatchRateOnByte(), + rgConfig.getReplicationDispatchRateInBytes().longValue()); + Assert.assertEquals(r2Limiter.get().getDispatchRateOnMsg(), + rgConfig.getReplicationDispatchRateInMsgs().longValue()); + r1 = retRG.getMonitoredEntity(ResourceGroupMonitoringClass.ReplicationDispatch, "r1"); + Assert.assertEquals(r1.configValuesPerPeriod.bytes, newR1Limiter.getDispatchThrottlingRateInByte()); + Assert.assertEquals(r1.configValuesPerPeriod.messages, newR1Limiter.getDispatchThrottlingRateInMsg()); + + // r2 + DispatchRate newR2Limiter = + DispatchRate.builder().dispatchThrottlingRateInMsg(2048).dispatchThrottlingRateInByte(400).build(); + replicatorLimiterMap.put(retRG.getReplicatorDispatchRateLimiterKey("r2"), newR2Limiter); + rgs.resourceGroupUpdate(rgName, rgConfig); + + Assert.assertNotEquals(r1Limiter.get(), r2Limiter.get()); + Assert.assertEquals(r1Limiter.get().getDispatchRateOnByte(), newR1Limiter.getDispatchThrottlingRateInByte()); + Assert.assertEquals(r1Limiter.get().getDispatchRateOnMsg(), newR1Limiter.getDispatchThrottlingRateInMsg()); + Assert.assertEquals(r2Limiter.get().getDispatchRateOnByte(), newR2Limiter.getDispatchThrottlingRateInByte()); + Assert.assertEquals(r2Limiter.get().getDispatchRateOnMsg(), newR2Limiter.getDispatchThrottlingRateInMsg()); + + PerMonitoringClassFields r2 = retRG.getMonitoredEntity(ResourceGroupMonitoringClass.ReplicationDispatch, "r2"); + Assert.assertEquals(r2.configValuesPerPeriod.bytes, newR2Limiter.getDispatchThrottlingRateInByte()); + Assert.assertEquals(r2.configValuesPerPeriod.messages, newR2Limiter.getDispatchThrottlingRateInMsg()); + + // remove r1 + replicatorLimiterMap.remove(retRG.getReplicatorDispatchRateLimiterKey("r1")); + rgs.resourceGroupUpdate(rgName, rgConfig); + Assert.assertEquals(r1Limiter.get().getDispatchRateOnByte(), + rgConfig.getReplicationDispatchRateInBytes().longValue()); + Assert.assertEquals(r1Limiter.get().getDispatchRateOnMsg(), + rgConfig.getReplicationDispatchRateInMsgs().longValue()); + Assert.assertEquals(r2Limiter.get().getDispatchRateOnByte(), newR2Limiter.getDispatchThrottlingRateInByte()); + Assert.assertEquals(r2Limiter.get().getDispatchRateOnMsg(), newR2Limiter.getDispatchThrottlingRateInMsg()); + + r1 = retRG.getMonitoredEntity(ResourceGroupMonitoringClass.ReplicationDispatch, "r1"); + Assert.assertEquals(r1.configValuesPerPeriod.bytes, + rgConfig.getReplicationDispatchRateInBytes().longValue()); + Assert.assertEquals(r1.configValuesPerPeriod.messages, + rgConfig.getReplicationDispatchRateInMsgs().intValue()); + + // remove r2 + replicatorLimiterMap.remove(retRG.getReplicatorDispatchRateLimiterKey("r2")); + rgs.resourceGroupUpdate(rgName, rgConfig); + Assert.assertEquals(r1Limiter.get().getDispatchRateOnByte(), + rgConfig.getReplicationDispatchRateInBytes().longValue()); + Assert.assertEquals(r1Limiter.get().getDispatchRateOnMsg(), + rgConfig.getReplicationDispatchRateInMsgs().longValue()); + Assert.assertEquals(r2Limiter.get().getDispatchRateOnByte(), + rgConfig.getReplicationDispatchRateInBytes().longValue()); + Assert.assertEquals(r2Limiter.get().getDispatchRateOnMsg(), + rgConfig.getReplicationDispatchRateInMsgs().longValue()); + + r2 = retRG.getMonitoredEntity(ResourceGroupMonitoringClass.ReplicationDispatch, "r2"); + Assert.assertEquals(r2.configValuesPerPeriod.bytes, + rgConfig.getReplicationDispatchRateInBytes().longValue()); + Assert.assertEquals(r2.configValuesPerPeriod.messages, + rgConfig.getReplicationDispatchRateInMsgs().intValue()); + + rgs.resourceGroupDelete(rgName); + + // Check the rate limiters are removed. + Assert.assertNull(r1Limiter.get()); + Assert.assertNull(r2Limiter.get()); + } + + @Test public void testResourceGroupOps() throws PulsarAdminException, InterruptedException { org.apache.pulsar.common.policies.data.ResourceGroup rgConfig = new org.apache.pulsar.common.policies.data.ResourceGroup(); @@ -152,6 +294,8 @@ public void testResourceGroupOps() throws PulsarAdminException, InterruptedExcep rgConfig.setPublishRateInMsgs(100); rgConfig.setDispatchRateInBytes(40000L); rgConfig.setDispatchRateInMsgs(500); + rgConfig.setReplicationDispatchRateInBytes(2000L); + rgConfig.setReplicationDispatchRateInMsgs(400L); int initialNumQuotaCalculations = numAnonymousQuotaCalculations; rgs.resourceGroupCreate(rgName, rgConfig); @@ -162,27 +306,33 @@ public void testResourceGroupOps() throws PulsarAdminException, InterruptedExcep new org.apache.pulsar.common.policies.data.ResourceGroup(); Assert.assertThrows(PulsarAdminException.class, () -> rgs.resourceGroupUpdate(randomRgName, randomConfig)); - rgConfig.setPublishRateInBytes(rgConfig.getPublishRateInBytes() * 10); - rgConfig.setPublishRateInMsgs(rgConfig.getPublishRateInMsgs() * 10); - rgConfig.setDispatchRateInBytes(rgConfig.getDispatchRateInBytes() / 10); - rgConfig.setDispatchRateInMsgs(rgConfig.getDispatchRateInMsgs() / 10); + rgConfig.setPublishRateInBytes(rgConfig.getPublishRateInBytes()*10); + rgConfig.setPublishRateInMsgs(rgConfig.getPublishRateInMsgs()*10); + rgConfig.setDispatchRateInBytes(rgConfig.getDispatchRateInBytes()/10); + rgConfig.setDispatchRateInMsgs(rgConfig.getDispatchRateInMsgs()/10); + rgConfig.setReplicationDispatchRateInBytes(rgConfig.getReplicationDispatchRateInBytes()/10); + rgConfig.setReplicationDispatchRateInMsgs(rgConfig.getReplicationDispatchRateInMsgs()/10); rgs.resourceGroupUpdate(rgName, rgConfig); Assert.assertEquals(rgs.getNumResourceGroups(), 1); - ResourceGroup retRG = rgs.resourceGroupGet(randomRgName); - Assert.assertNull(retRG); + Assert.assertNull(rgs.resourceGroupGet(randomRgName)); - retRG = rgs.resourceGroupGet(rgName); + ResourceGroup retRG = rgs.resourceGroupGet(rgName); Assert.assertNotEquals(retRG, null); PerMonitoringClassFields monClassFields; - monClassFields = retRG.monitoringClassFields[ResourceGroupMonitoringClass.Publish.ordinal()]; + monClassFields = retRG.getMonitoredEntity(ResourceGroupMonitoringClass.Publish, null); Assert.assertEquals(monClassFields.configValuesPerPeriod.bytes, rgConfig.getPublishRateInBytes().longValue()); Assert.assertEquals(monClassFields.configValuesPerPeriod.messages, rgConfig.getPublishRateInMsgs().intValue()); - monClassFields = retRG.monitoringClassFields[ResourceGroupMonitoringClass.Dispatch.ordinal()]; + monClassFields = retRG.getMonitoredEntity(ResourceGroupMonitoringClass.Dispatch, null); Assert.assertEquals(monClassFields.configValuesPerPeriod.bytes, rgConfig.getDispatchRateInBytes().longValue()); Assert.assertEquals(monClassFields.configValuesPerPeriod.messages, rgConfig.getDispatchRateInMsgs().intValue()); + monClassFields = retRG.getMonitoredEntity(ResourceGroupMonitoringClass.ReplicationDispatch, "r1"); + Assert.assertEquals(monClassFields.configValuesPerPeriod.bytes, + rgConfig.getReplicationDispatchRateInBytes().longValue()); + Assert.assertEquals(monClassFields.configValuesPerPeriod.messages, + rgConfig.getReplicationDispatchRateInMsgs().intValue()); Assert.assertThrows(PulsarAdminException.class, () -> rgs.resourceGroupDelete(randomRgName)); @@ -196,6 +346,7 @@ public void testResourceGroupOps() throws PulsarAdminException, InterruptedExcep final NamespaceName tenantAndNamespace = NamespaceName.get(tenantName, namespaceName); rgs.registerNameSpace(rgName, tenantAndNamespace); + rgs.registerTopic(rgName, topic); // Delete of our valid config should throw until we unref correspondingly. Assert.assertThrows(PulsarAdminException.class, () -> rgs.resourceGroupDelete(rgName)); @@ -203,30 +354,23 @@ public void testResourceGroupOps() throws PulsarAdminException, InterruptedExcep // Attempt to report for a few rounds (simulating a fill usage with transport mgr). // It should say "we need to report now" every 'maxUsageReportSuppressRounds' rounds, // even if usage does not change. - final ResourceUsage usage = new ResourceUsage(); - NetworkUsage nwUsage; - - boolean needToReport; - for (int monClassIdx = 0; monClassIdx < ResourceGroupMonitoringClass.values().length; monClassIdx++) { - ResourceGroupMonitoringClass monClass = ResourceGroupMonitoringClass.values()[monClassIdx]; - // Gross hack! - if (monClass == ResourceGroupMonitoringClass.Publish) { - nwUsage = usage.setPublish(); - } else { - nwUsage = usage.setDispatch(); - } - // We know that dummyQuotaCalc::needToReportLocalUsage() makes us report usage once every - // maxUsageReportSuppressRounds iterations. So, if we run for maxUsageReportSuppressRounds iterations, - // we should see needToReportLocalUsage() return true at least once. - Set myBoolSet = new HashSet<>(); - for (int idx = 0; idx < ResourceGroupService.MaxUsageReportSuppressRounds; idx++) { - needToReport = retRG.setUsageInMonitoredEntity(monClass, nwUsage); - myBoolSet.add(needToReport); + retRG.getMonitoringClassFieldsMap().forEach((k, v) -> { + List usageList = new ArrayList<>(); + for (int idx = 0; idx < conf.getResourceUsageMaxUsageReportSuppressRounds(); idx++) { + ResourceUsage usage = new ResourceUsage(); + usageList.add(usage); + retRG.setUsageInMonitoredEntity(usage); } // Expect to see at least one true and at least one false. - Assert.assertTrue(myBoolSet.contains(true)); - Assert.assertTrue(myBoolSet.contains(false)); - } + Assert.assertTrue(usageList.stream() + .anyMatch(n -> n.hasPublish() || n.hasDispatch() || n.getReplicatorsCount() != 0)); + Assert.assertTrue(usageList.stream() + .anyMatch(n -> (!n.hasPublish()) && (!n.hasDispatch()) && (n.getReplicatorsCount() == 0))); + }); + + rgs.unRegisterTenant(rgName, tenantName); + rgs.unRegisterNameSpace(rgName, tenantAndNamespace); + rgs.unRegisterTopic(topic); BytesAndMessagesCount publishQuota = rgs.getPublishRateLimiters(rgName); @@ -255,6 +399,101 @@ public void testResourceGroupOps() throws PulsarAdminException, InterruptedExcep Assert.assertThrows(PulsarAdminException.class, () -> rgs.getPublishRateLimiters(rgName)); Assert.assertEquals(rgs.getNumResourceGroups(), 0); + + Assert.assertEquals(rgs.getTopicConsumeStats().estimatedSize(), 0); + Assert.assertEquals(rgs.getTopicProduceStats().estimatedSize(), 0); + Assert.assertEquals(rgs.getReplicationDispatchStats().estimatedSize(), 0); + Assert.assertEquals(rgs.getTopicToReplicatorsMap().size(), 0); + } + + @Test + public void testCleanupStatsWhenUnRegisterTopic() + throws PulsarAdminException { + String tenantName = UUID.randomUUID().toString(); + org.apache.pulsar.common.policies.data.ResourceGroup rgConfig = + new org.apache.pulsar.common.policies.data.ResourceGroup(); + final String rgName = UUID.randomUUID().toString(); + rgConfig.setPublishRateInBytes(15000L); + rgConfig.setPublishRateInMsgs(100); + rgConfig.setDispatchRateInBytes(40000L); + rgConfig.setDispatchRateInMsgs(500); + rgConfig.setReplicationDispatchRateInBytes(2000L); + rgConfig.setReplicationDispatchRateInMsgs(400L); + + rgs.resourceGroupCreate(rgName, rgConfig); + String nsName = tenantName + "/" + UUID.randomUUID(); + TopicName topicName = TopicName.get(nsName + "/" + UUID.randomUUID()); + String topic = topicName.toString(); + + rgs.registerTopic(rgName, topicName); + + // Simulate replicator + rgs.updateStatsWithDiff(topic, "remote-cluster", tenantName, nsName, 1, 1, + ResourceGroupMonitoringClass.ReplicationDispatch); + rgs.updateStatsWithDiff(topic, null, tenantName, nsName, 1, 1, + ResourceGroupMonitoringClass.Publish); + rgs.updateStatsWithDiff(topic, null, tenantName, nsName, 1, 1, + ResourceGroupMonitoringClass.Dispatch); + Assert.assertEquals(rgs.getTopicProduceStats().asMap().size(), 1); + Assert.assertEquals(rgs.getTopicConsumeStats().asMap().size(), 1); + Assert.assertEquals(rgs.getReplicationDispatchStats().asMap().size(), 1); + Assert.assertEquals(rgs.getTopicToReplicatorsMap().size(), 1); + Set replicators = rgs.getTopicToReplicatorsMap().get(rgs.getTopicToReplicatorsMap().keys().nextElement()); + Assert.assertEquals(replicators.size(), 1); + + rgs.unRegisterTopic(TopicName.get(topic)); + + Assert.assertEquals(rgs.getTopicProduceStats().asMap().size(), 0); + Assert.assertEquals(rgs.getTopicConsumeStats().asMap().size(), 0); + Assert.assertEquals(rgs.getReplicationDispatchStats().asMap().size(), 0); + Assert.assertEquals(rgs.getReplicationDispatchStats().asMap().size(), 0); + Assert.assertEquals(rgs.getTopicToReplicatorsMap().size(), 0); + + rgs.resourceGroupDelete(rgName); + } + + @Test + public void testCleanupStatsWhenUnRegisterNamespace() + throws PulsarAdminException { + String tenantName = UUID.randomUUID().toString(); + org.apache.pulsar.common.policies.data.ResourceGroup rgConfig = + new org.apache.pulsar.common.policies.data.ResourceGroup(); + final String rgName = UUID.randomUUID().toString(); + rgConfig.setPublishRateInBytes(15000L); + rgConfig.setPublishRateInMsgs(100); + rgConfig.setDispatchRateInBytes(40000L); + rgConfig.setDispatchRateInMsgs(500); + rgConfig.setReplicationDispatchRateInBytes(2000L); + rgConfig.setReplicationDispatchRateInMsgs(400L); + + rgs.resourceGroupCreate(rgName, rgConfig); + String nsName = tenantName + "/" + UUID.randomUUID(); + TopicName topicName = TopicName.get(nsName + "/" + UUID.randomUUID()); + String topic = topicName.toString(); + + rgs.registerNameSpace(rgName, topicName.getNamespaceObject()); + + // Simulate replicator + rgs.updateStatsWithDiff(topic, "remote-cluster", tenantName, nsName, 1, 1, + ResourceGroupMonitoringClass.ReplicationDispatch); + rgs.updateStatsWithDiff(topic, null, tenantName, nsName, 1, 1, + ResourceGroupMonitoringClass.Publish); + rgs.updateStatsWithDiff(topic, null, tenantName, nsName, 1, 1, + ResourceGroupMonitoringClass.Dispatch); + Assert.assertEquals(rgs.getTopicProduceStats().asMap().size(), 1); + Assert.assertEquals(rgs.getTopicConsumeStats().asMap().size(), 1); + Assert.assertEquals(rgs.getReplicationDispatchStats().asMap().size(), 1); + Set replicators = rgs.getTopicToReplicatorsMap().get(rgs.getTopicToReplicatorsMap().keys().nextElement()); + Assert.assertEquals(replicators.size(), 1); + + rgs.unRegisterNameSpace(rgName, topicName.getNamespaceObject()); + + Assert.assertEquals(rgs.getTopicProduceStats().asMap().size(), 0); + Assert.assertEquals(rgs.getTopicConsumeStats().asMap().size(), 0); + Assert.assertEquals(rgs.getReplicationDispatchStats().asMap().size(), 0); + Assert.assertEquals(rgs.getTopicToReplicatorsMap().size(), 0); + + rgs.resourceGroupDelete(rgName); } /** @@ -300,6 +539,254 @@ public void testClose() throws Exception { "SchedulersRunning flag should be false after close()."); } + private void assertTopicStatsCache(Cache cache, long durationMS) { + Optional> expirationOptional = + cache.policy().expireAfterAccess(); + Assert.assertTrue(expirationOptional.isPresent()); + Expiration expiration = expirationOptional.get(); + Assert.assertEquals(expiration.getExpiresAfter().toMillis(), durationMS); + } + + @Test + public void testTopicStatsCache() { + long ms = 2_000; + Cache cache = + pulsar.getResourceGroupServiceManager().newStatsCache(TimeUnit.MILLISECONDS.toMillis(ms)); + String key = "topic-1"; + BytesAndMessagesCount value = new BytesAndMessagesCount(); + cache.put(key, value); + Assert.assertEquals(cache.getIfPresent(key), value); + Awaitility.await().pollDelay(ms + 200 , TimeUnit.MILLISECONDS).untilAsserted(() -> { + Assert.assertNull(cache.getIfPresent(key)); + }); + + long expMS = + TimeUnit.SECONDS.toMillis(pulsar.getConfiguration().getResourceUsageTransportPublishIntervalInSecs()) + * 2; + assertTopicStatsCache(pulsar.getResourceGroupServiceManager().getTopicConsumeStats(), expMS); + assertTopicStatsCache(pulsar.getResourceGroupServiceManager().getTopicProduceStats(), expMS); + assertTopicStatsCache(pulsar.getResourceGroupServiceManager().getReplicationDispatchStats(), expMS); + } + + @Test + public void testBrokerStatsCache() throws PulsarAdminException { + long ms = 2_000; + String key = "broker-1"; + PerBrokerUsageStats value = new PerBrokerUsageStats(); + PerMonitoringClassFields perMonitoringClassFields = PerMonitoringClassFields.create(ms); + Cache usageFromOtherBrokers = perMonitoringClassFields.usageFromOtherBrokers; + usageFromOtherBrokers.put(key, value); + Assert.assertEquals(usageFromOtherBrokers.getIfPresent(key), value); + Awaitility.await().atMost(ms + 500, TimeUnit.MILLISECONDS).untilAsserted(() -> { + Assert.assertNull(usageFromOtherBrokers.getIfPresent(key)); + }); + + org.apache.pulsar.common.policies.data.ResourceGroup rgConfig = + new org.apache.pulsar.common.policies.data.ResourceGroup(); + final String rgName = UUID.randomUUID().toString(); + rgConfig.setPublishRateInBytes(15000L); + rgConfig.setPublishRateInMsgs(100); + rgConfig.setDispatchRateInBytes(40000L); + rgConfig.setDispatchRateInMsgs(500); + rgConfig.setReplicationDispatchRateInBytes(2000L); + rgConfig.setReplicationDispatchRateInMsgs(400L); + + ResourceGroupService resourceGroupServiceManager = pulsar.getResourceGroupServiceManager(); + resourceGroupServiceManager.resourceGroupCreate(rgName, rgConfig); + ResourceGroup resourceGroup = resourceGroupServiceManager.resourceGroupGet(rgName); + PerMonitoringClassFields publishMonitoredEntity = + resourceGroup.getMonitoredEntity(ResourceGroupMonitoringClass.Publish, null); + Cache cache = publishMonitoredEntity.usageFromOtherBrokers; + Optional> expirationOptional = + cache.policy().expireAfterWrite(); + Assert.assertTrue(expirationOptional.isPresent()); + Expiration brokerUsageStatsExpiration = expirationOptional.get(); + + long statsDuration = + TimeUnit.SECONDS.toMillis(pulsar.getConfiguration().getResourceUsageTransportPublishIntervalInSecs()) + * conf.getResourceUsageMaxUsageReportSuppressRounds() * 2; + Assert.assertEquals(brokerUsageStatsExpiration.getExpiresAfter().toMillis(), statsDuration); + } + + protected String getReplicatorDispatchRateLimiterKey(String remoteCluster) { + return DispatchRateLimiter.getReplicatorDispatchRateKey(pulsar.getConfiguration().getClusterName(), + remoteCluster); + } + + @Test + public void testPublishAndDispatchQuotaCalculation() throws PulsarAdminException { + // Mocks and setup + ResourceUsageTransportManager resourceUsageTransportManager = mock(ResourceUsageTransportManager.class); + ResourceGroupService resourceGroupService = new ResourceGroupService( + pulsar, TimeUnit.HOURS, resourceUsageTransportManager, + new ResourceQuotaCalculatorImpl(pulsar) + ); + + // Setup: Resource Group Configuration + String rg1Name = UUID.randomUUID().toString(); + org.apache.pulsar.common.policies.data.ResourceGroup rg1Config = + new org.apache.pulsar.common.policies.data.ResourceGroup(); + rg1Config.setPublishRateInBytes(100L); + rg1Config.setPublishRateInMsgs(200); + rg1Config.setDispatchRateInBytes(40000L); + rg1Config.setDispatchRateInMsgs(500); + + resourceGroupService.resourceGroupCreate(rg1Name, rg1Config); + ResourceGroup rg1Ref = resourceGroupService.resourceGroupGet(rg1Name); + + // Verify initial publish limiter values + ResourceGroupPublishLimiter publishLimiter = rg1Ref.getResourceGroupPublishLimiter(); + Assert.assertEquals(publishLimiter.getResourceGroupPublishValues().bytes, 100L); + Assert.assertEquals(publishLimiter.getResourceGroupPublishValues().messages, 200); + + // Verify initial dispatch limiter values + ResourceGroupDispatchLimiter dispatchLimiter = rg1Ref.getResourceGroupDispatchLimiter(); + Assert.assertEquals(dispatchLimiter.getDispatchRateOnByte(), 40000L); + Assert.assertEquals(dispatchLimiter.getDispatchRateOnMsg(), 500); + + // Simulate local usage + ResourceUsage localUsage = new ResourceUsage(); + // Local dispatch usage + localUsage.setDispatch().setBytesPerPeriod(100).setMessagesPerPeriod(200); + // Local publish usage + localUsage.setPublish().setBytesPerPeriod(50).setMessagesPerPeriod(150); + + // Simulate remote usage + ResourceUsage remoteUsage = new ResourceUsage(); + // Remote dispatch usage + remoteUsage.setDispatch().setBytesPerPeriod(500).setMessagesPerPeriod(800); + // Remote publish usage + remoteUsage.setPublish().setBytesPerPeriod(250).setMessagesPerPeriod(600); + + // Report usages + rg1Ref.getUsageFromMonitoredEntity(remoteUsage, "pulsar://broker-2:6650"); + rg1Ref.getUsageFromMonitoredEntity(localUsage, pulsar.getBrokerServiceUrl()); + + // Calculate publish quota + // Global limits: messages = 200, bytes = 100 + // Usage: + // - Local: messages = 150, bytes = 50 + // - Remote: messages = 600, bytes = 250 + // Quota: + // - Messages: 150 / (150 + 600) * 200 ≈ 40 + // - Bytes: 50 / (50 + 250) * 100 ≈ 16 + resourceGroupService.calculateQuotaByMonClass(rg1Name, rg1Ref, ResourceGroupMonitoringClass.Publish); + publishLimiter = rg1Ref.getResourceGroupPublishLimiter(); + Assert.assertEquals(publishLimiter.getResourceGroupPublishValues().bytes, 16); + Assert.assertEquals(publishLimiter.getResourceGroupPublishValues().messages, 40); + + // Calculate dispatch quota + // Global limits: messages = 500, bytes = 40000 + // Usage: + // - Local: messages = 200, bytes = 100 + // - Remote: messages = 800, bytes = 500 + // Quota: + // - Messages: 200 / (200 + 800) * 500 = 100 + // - Bytes: 100 / (100 + 500) * 40000 ≈ 6666 + resourceGroupService.calculateQuotaByMonClass(rg1Name, rg1Ref, ResourceGroupMonitoringClass.Dispatch); + dispatchLimiter = rg1Ref.getResourceGroupDispatchLimiter(); + Assert.assertEquals(dispatchLimiter.getDispatchRateOnByte(), 6666); + Assert.assertEquals(dispatchLimiter.getDispatchRateOnMsg(), 100); + } + + @Test + public void testReplicationDispatchQuotaCalculation() throws PulsarAdminException { + ResourceUsageTransportManager transportManager = mock(ResourceUsageTransportManager.class); + ResourceGroupService resourceGroupService = new ResourceGroupService( + pulsar, TimeUnit.HOURS, transportManager, new ResourceQuotaCalculatorImpl(pulsar)); + + String rgName = UUID.randomUUID().toString(); + org.apache.pulsar.common.policies.data.ResourceGroup rgConfig = new org.apache.pulsar.common.policies.data.ResourceGroup(); + rgConfig.setReplicationDispatchRateInBytes(2000L); + rgConfig.setReplicationDispatchRateInMsgs(400L); + + Map replicatorDispatchRateMap = new HashMap<>(); + replicatorDispatchRateMap.put(getReplicatorDispatchRateLimiterKey("r1"), DispatchRate.builder().dispatchThrottlingRateInByte(1000).dispatchThrottlingRateInMsg(300).build()); + replicatorDispatchRateMap.put(getReplicatorDispatchRateLimiterKey("r3"), DispatchRate.builder().dispatchThrottlingRateInByte(500).dispatchThrottlingRateInMsg(100).build()); + rgConfig.setReplicatorDispatchRate(replicatorDispatchRateMap); + + resourceGroupService.resourceGroupCreate(rgName, rgConfig); + ResourceGroup rg = resourceGroupService.resourceGroupGet(rgName); + + Map> limiters = new HashMap<>(); + for (String cluster : Lists.newArrayList("r1", "r2", "r3", "r4")) { + AtomicReference ref = new AtomicReference<>(); + rg.registerReplicatorDispatchRateLimiter(cluster, ref::set); + limiters.put(cluster, ref); + } + + // Local usage + ResourceUsage local = new ResourceUsage(); + local.addReplicator().setRemoteCluster("r1").setNetworkUsage().setBytesPerPeriod(100).setMessagesPerPeriod(100); + local.addReplicator().setRemoteCluster("r2").setNetworkUsage().setBytesPerPeriod(1400).setMessagesPerPeriod(90); + local.addReplicator().setRemoteCluster("r3").setNetworkUsage().setBytesPerPeriod(300).setMessagesPerPeriod(50); + local.addReplicator().setRemoteCluster("r4").setNetworkUsage().setBytesPerPeriod(100).setMessagesPerPeriod(10); + local.addReplicator().setRemoteCluster("r5").setNetworkUsage().setBytesPerPeriod(1900).setMessagesPerPeriod(100); + + // Remote usage + ResourceUsage remote = new ResourceUsage(); + remote.addReplicator().setRemoteCluster("r1").setNetworkUsage().setBytesPerPeriod(50).setMessagesPerPeriod(20); + remote.addReplicator().setRemoteCluster("r2").setNetworkUsage().setBytesPerPeriod(400).setMessagesPerPeriod(280); + remote.addReplicator().setRemoteCluster("r3").setNetworkUsage().setBytesPerPeriod(100).setMessagesPerPeriod(50); + remote.addReplicator().setRemoteCluster("r4").setNetworkUsage().setBytesPerPeriod(200).setMessagesPerPeriod(80); + remote.addReplicator().setRemoteCluster("r5").setNetworkUsage().setBytesPerPeriod(100).setMessagesPerPeriod(40); + + rg.getUsageFromMonitoredEntity(remote, "pulsar://broker-remote"); + rg.getUsageFromMonitoredEntity(local, pulsar.getBrokerServiceUrl()); + + resourceGroupService.calculateQuotaByMonClass(rgName, rg, ResourceGroupMonitoringClass.ReplicationDispatch); + + // r1 (specific) + // Global limits: 1000 bytes, 300 messages + // Local usage: 100 B / 100 M + // Remote usage: 50 B / 20 M + // Quota calculation: + // 100 / (100 + 50) * 1000 = 666.67 (≈ 666) B + // 100 / (100 + 20) * 300 = 250 (M) + Assert.assertEquals(limiters.get("r1").get().getDispatchRateOnByte(), 666); + Assert.assertEquals(limiters.get("r1").get().getDispatchRateOnMsg(), 250); + + // r3 (specific) + // Global limits: 500 bytes, 100 messages + // Local usage: 300 B / 50 M + // Remote usage: 100 B / 50 M + // Quota calculation: + // 300 / (300 + 100) * 500 = 375 (B) + // 50 / (50 + 50) * 100 = 50 (M) + Assert.assertEquals(limiters.get("r3").get().getDispatchRateOnByte(), 375); + Assert.assertEquals(limiters.get("r3").get().getDispatchRateOnMsg(), 50); + + // r2, r4 (global) + // Local usage: + // r2: 1400 B / 90 M + // r4: 100 B / 10 M + // r5: 1900 B / 100 M + // Remote usage: + // r2: 400 B / 280 M + // r4: 200 B / 80 M + // r5: 100 B / 40 M + // Total usage: + // r2: 1400 + 400 = 1800 B / 90 + 280 = 370 M + // r4: 100 + 200 = 300 B / 10 + 80 = 90 M + // r5: 1900 + 100 = 2000 B / 100 + 40 = 140 M + + // long totalBytesLocal = 1400 + 100 + 1900; // 3400 + // long totalMsgsLocal = 90 + 10 + 100; // 200 + // long totalBytes= 1800 + 300 + 2000; // 4100 + // long totalMsgs = 370 + 90 + 140; // 600 + + // Global limit: 2000 bytes, 400 messages + // Quota calculation: + // 3400 / 4100 * 2000 = 1658.54 (≈ 1658) B + // 200 / 600 * 400 = 133.33 (≈ 133) M + for (String cluster : Lists.newArrayList("r2", "r4")) { + ResourceGroupDispatchLimiter limiter = limiters.get(cluster).get(); + Assert.assertNotNull(limiter); + Assert.assertEquals(limiter.getDispatchRateOnByte(), 1658); + Assert.assertEquals(limiter.getDispatchRateOnMsg(), 133); + } + } + private ResourceGroupService rgs; int numAnonymousQuotaCalculations; @@ -307,7 +794,6 @@ public void testClose() throws Exception { private static final int PUBLISH_INTERVAL_SECS = 500; private void prepareData() throws PulsarAdminException { - this.conf.setResourceUsageTransportPublishIntervalInSecs(PUBLISH_INTERVAL_SECS); admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); } /** diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationOnTopicLevelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationOnTopicLevelTest.java new file mode 100644 index 0000000000000..07809c01738ac --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationOnTopicLevelTest.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.resourcegroup; + +import static org.testng.Assert.assertNotNull; +import com.google.common.collect.Sets; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.resourcegroup.ResourceGroup.BytesAndMessagesCount; +import org.apache.pulsar.broker.resourcegroup.ResourceGroup.ResourceGroupMonitoringClass; +import org.apache.pulsar.broker.resourcegroup.ResourceGroupService.ResourceGroupUsageStatsType; +import org.apache.pulsar.broker.service.BrokerService; +import org.apache.pulsar.broker.service.resource.usage.ResourceUsage; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; +import org.awaitility.Awaitility; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Slf4j +public class ResourceGroupUsageAggregationOnTopicLevelTest extends ProducerConsumerBase { + + private final String TenantName = "pulsar-test"; + private final String NsName = "test"; + private final String TenantAndNsName = TenantName + "/" + NsName; + private final String TestProduceConsumeTopicName = "/test/prod-cons-topic"; + private final String PRODUCE_CONSUME_PERSISTENT_TOPIC = "persistent://" + TenantAndNsName + TestProduceConsumeTopicName; + private final String PRODUCE_CONSUME_NON_PERSISTENT_TOPIC = + "non-persistent://" + TenantAndNsName + TestProduceConsumeTopicName; + + @BeforeMethod + @Override + protected void setup() throws Exception { + super.internalSetup(); + this.conf.setAllowAutoTopicCreation(true); + + final String clusterName = "test"; + admin.clusters().createCluster(clusterName, ClusterData.builder().serviceUrl(brokerUrl.toString()).build()); + admin.tenants().createTenant(TenantName, + new TenantInfoImpl(Sets.newHashSet("fakeAdminRole"), Sets.newHashSet(clusterName))); + admin.namespaces().createNamespace(TenantAndNsName); + admin.namespaces().setNamespaceReplicationClusters(TenantAndNsName, Sets.newHashSet(clusterName)); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testPersistentTopicProduceConsumeUsageOnRG() throws Exception { + testProduceConsumeUsageOnRG(PRODUCE_CONSUME_PERSISTENT_TOPIC); + } + + @Test + public void testNonPersistentTopicProduceConsumeUsageOnRG() throws Exception { + testProduceConsumeUsageOnRG(PRODUCE_CONSUME_NON_PERSISTENT_TOPIC); + } + + private void testProduceConsumeUsageOnRG(String topicString) throws Exception { + ResourceQuotaCalculator dummyQuotaCalc = new ResourceQuotaCalculator() { + @Override + public boolean needToReportLocalUsage(long currentBytesUsed, long lastReportedBytes, + long currentMessagesUsed, long lastReportedMessages, + long lastReportTimeMSecsSinceEpoch) { + return false; + } + + @Override + public long computeLocalQuota(long confUsage, long myUsage, long[] allUsages) { + return 0; + } + }; + + @Cleanup + ResourceUsageTopicTransportManager transportMgr = new ResourceUsageTopicTransportManager(pulsar); + @Cleanup + ResourceGroupService rgs = new ResourceGroupService(pulsar, TimeUnit.MILLISECONDS, transportMgr, + dummyQuotaCalc); + + String activeRgName = "runProduceConsume"; + ResourceGroup activeRG; + + ResourceUsagePublisher ruP = new ResourceUsagePublisher() { + @Override + public String getID() { + return rgs.resourceGroupGet(activeRgName).resourceGroupName; + } + + @Override + public void fillResourceUsage(ResourceUsage resourceUsage) { + rgs.resourceGroupGet(activeRgName).rgFillResourceUsage(resourceUsage); + } + }; + + ResourceUsageConsumer ruC = new ResourceUsageConsumer() { + @Override + public String getID() { + return rgs.resourceGroupGet(activeRgName).resourceGroupName; + } + + @Override + public void acceptResourceUsage(String broker, ResourceUsage resourceUsage) { + rgs.resourceGroupGet(activeRgName).rgResourceUsageListener(broker, resourceUsage); + } + }; + + org.apache.pulsar.common.policies.data.ResourceGroup rgConfig = + new org.apache.pulsar.common.policies.data.ResourceGroup(); + rgConfig.setPublishRateInBytes(1500L); + rgConfig.setPublishRateInMsgs(100); + rgConfig.setDispatchRateInBytes(4000L); + rgConfig.setPublishRateInMsgs(500); + + rgs.resourceGroupCreate(activeRgName, rgConfig, ruP, ruC); + + activeRG = rgs.resourceGroupGet(activeRgName); + assertNotNull(activeRG); + + String subscriptionName = "my-subscription"; + @Cleanup + Consumer consumer = pulsarClient.newConsumer() + .topic(topicString) + .subscriptionName(subscriptionName) + .subscriptionType(SubscriptionType.Shared) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topicString) + .create(); + + TopicName myTopic = TopicName.get(topicString); + rgs.unRegisterTopic(myTopic); + rgs.registerTopic(activeRgName,myTopic); + + final int NumMessagesToSend = 10; + int sentNumBytes = 0; + int sentNumMsgs = 0; + for (int ix = 0; ix < NumMessagesToSend; ix++) { + byte[] mesg = String.format("Hi, ix=%s", ix).getBytes(); + producer.send(mesg); + sentNumBytes += mesg.length; + sentNumMsgs++; + } + + this.verifyStats(rgs, topicString, activeRgName, sentNumBytes, sentNumMsgs, 0, 0, + true, false); + + int recvdNumBytes = 0; + int recvdNumMsgs = 0; + + Message message; + while (recvdNumMsgs < sentNumMsgs) { + message = consumer.receive(); + recvdNumBytes += message.getValue().length; + recvdNumMsgs++; + } + + this.verifyStats(rgs,topicString, activeRgName, sentNumBytes, sentNumMsgs, recvdNumBytes, recvdNumMsgs, + true, true); + } + + private void verifyStats(ResourceGroupService rgs, String topicString, String rgName, + int sentNumBytes, int sentNumMsgs, + int recvdNumBytes, int recvdNumMsgs, + boolean checkProduce, boolean checkConsume) throws PulsarAdminException { + BrokerService bs = pulsar.getBrokerService(); + Awaitility.await().untilAsserted(() -> { + TopicStatsImpl topicStats = bs.getTopicStats().get(topicString); + assertNotNull(topicStats); + if (checkProduce) { + Assert.assertTrue(topicStats.bytesInCounter >= sentNumBytes); + Assert.assertEquals(sentNumMsgs, topicStats.msgInCounter); + } + if (checkConsume) { + Assert.assertTrue(topicStats.bytesOutCounter >= recvdNumBytes); + Assert.assertEquals(recvdNumMsgs, topicStats.msgOutCounter); + } + }); + if (sentNumMsgs > 0 || recvdNumMsgs > 0) { + rgs.aggregateResourceGroupLocalUsages(); + BytesAndMessagesCount prodCounts = rgs.getRGUsage(rgName, ResourceGroupMonitoringClass.Publish, + ResourceGroupUsageStatsType.Cumulative).entrySet().iterator().next().getValue(); + BytesAndMessagesCount consCounts = rgs.getRGUsage(rgName, ResourceGroupMonitoringClass.Dispatch, + ResourceGroupUsageStatsType.Cumulative).entrySet().iterator().next().getValue(); + + if (checkProduce) { + Assert.assertTrue(prodCounts.bytes >= sentNumBytes); + Assert.assertEquals(sentNumMsgs, prodCounts.messages); + } + if (checkConsume) { + Assert.assertTrue(consCounts.bytes >= recvdNumBytes); + Assert.assertEquals(recvdNumMsgs, consCounts.messages); + } + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationTest.java index 94622d17c4c55..fc1ba307a1d77 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationTest.java @@ -26,7 +26,6 @@ import org.apache.pulsar.broker.resourcegroup.ResourceGroup.BytesAndMessagesCount; import org.apache.pulsar.broker.resourcegroup.ResourceGroup.ResourceGroupMonitoringClass; import org.apache.pulsar.broker.resourcegroup.ResourceGroupService.ResourceGroupUsageStatsType; -import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.service.resource.usage.ResourceUsage; @@ -41,16 +40,15 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; -import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; -import org.awaitility.Awaitility; +import org.apache.pulsar.common.policies.data.TopicStats; import org.testng.Assert; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @Slf4j public class ResourceGroupUsageAggregationTest extends ProducerConsumerBase { - @BeforeClass + @BeforeMethod @Override protected void setup() throws Exception { super.internalSetup(); @@ -74,19 +72,23 @@ public long computeLocalQuota(long confUsage, long myUsage, long[] allUsages) { this.rgs = new ResourceGroupService(pulsar, TimeUnit.MILLISECONDS, transportMgr, dummyQuotaCalc); } - @AfterClass(alwaysRun = true) + @AfterMethod(alwaysRun = true) @Override protected void cleanup() throws Exception { super.internalCleanup(); } @Test - public void testProduceConsumeUsageOnRG() throws Exception { - testProduceConsumeUsageOnRG(produceConsumePersistentTopic); - testProduceConsumeUsageOnRG(produceConsumeNonPersistentTopic); + public void testProduceConsumeUsageOnRGWithPersistentTopic() throws Exception { + testProduceConsumeUsageOnRG(produceConsumePersistentTopic, "rg-" + System.currentTimeMillis()); } - private void testProduceConsumeUsageOnRG(String topicString) throws Exception { + @Test + public void testProduceConsumeUsageOnRGWithNonPersistentTopic() throws Exception { + testProduceConsumeUsageOnRG(produceConsumeNonPersistentTopic, "rg-" + System.currentTimeMillis()); + } + + private void testProduceConsumeUsageOnRG(String topicString, String activeRgName) throws Exception { ResourceUsagePublisher ruP = new ResourceUsagePublisher() { @Override public String getID() { @@ -120,6 +122,7 @@ public void acceptResourceUsage(String broker, ResourceUsage resourceUsage) { Producer producer = pulsarClient.newProducer() + .enableBatching(false) .topic(topicString) .create(); @@ -194,8 +197,6 @@ public void acceptResourceUsage(String broker, ResourceUsage resourceUsage) { } } } - rgs.getTopicConsumeStats().clear(); - rgs.getTopicProduceStats().clear(); rgs.unRegisterTenant(activeRgName, tenantString); rgs.unRegisterNameSpace(activeRgName, NamespaceName.get(nsString)); @@ -211,32 +212,29 @@ private void verifyStats(String topicString, String rgName, int recvdNumBytes, int recvdNumMsgs, boolean checkProduce, boolean checkConsume) throws InterruptedException, PulsarAdminException { - BrokerService bs = pulsar.getBrokerService(); - Awaitility.await().untilAsserted(() -> { - TopicStatsImpl topicStats = bs.getTopicStats().get(topicString); - Assert.assertNotNull(topicStats); - if (checkProduce) { - Assert.assertTrue(topicStats.bytesInCounter >= sentNumBytes); - Assert.assertEquals(sentNumMsgs, topicStats.msgInCounter); - } - if (checkConsume) { - Assert.assertTrue(topicStats.bytesOutCounter >= recvdNumBytes); - Assert.assertEquals(recvdNumMsgs, topicStats.msgOutCounter); - } - }); + TopicStats stats = admin.topics().getStats(topicString); + if (checkProduce) { + Assert.assertTrue(stats.getBytesInCounter() >= sentNumBytes); + Assert.assertEquals(sentNumMsgs, stats.getMsgInCounter()); + } + if (checkConsume) { + Assert.assertTrue(stats.getBytesOutCounter() >= recvdNumBytes); + Assert.assertEquals(recvdNumMsgs, stats.getMsgOutCounter()); + } + if (sentNumMsgs > 0 || recvdNumMsgs > 0) { rgs.aggregateResourceGroupLocalUsages(); // hack to ensure aggregator calculation without waiting BytesAndMessagesCount prodCounts = rgs.getRGUsage(rgName, ResourceGroupMonitoringClass.Publish, - ResourceGroupUsageStatsType.Cumulative); + ResourceGroupUsageStatsType.Cumulative).entrySet().iterator().next().getValue(); BytesAndMessagesCount consCounts = rgs.getRGUsage(rgName, ResourceGroupMonitoringClass.Dispatch, - ResourceGroupUsageStatsType.Cumulative); + ResourceGroupUsageStatsType.Cumulative).entrySet().iterator().next().getValue(); // Re-do the getRGUsage. // The counts should be equal, since there wasn't any intervening traffic on TEST_PRODUCE_CONSUME_TOPIC. BytesAndMessagesCount prodCounts1 = rgs.getRGUsage(rgName, ResourceGroupMonitoringClass.Publish, - ResourceGroupUsageStatsType.Cumulative); + ResourceGroupUsageStatsType.Cumulative).entrySet().iterator().next().getValue(); BytesAndMessagesCount consCounts1 = rgs.getRGUsage(rgName, ResourceGroupMonitoringClass.Dispatch, - ResourceGroupUsageStatsType.Cumulative); + ResourceGroupUsageStatsType.Cumulative).entrySet().iterator().next().getValue(); Assert.assertEquals(prodCounts1.bytes, prodCounts.bytes); Assert.assertEquals(prodCounts1.messages, prodCounts.messages); @@ -258,7 +256,6 @@ private void verifyStats(String topicString, String rgName, ResourceGroup activeRG; final org.apache.pulsar.common.policies.data.ResourceGroup rgConfig = new org.apache.pulsar.common.policies.data.ResourceGroup(); - final String activeRgName = "runProduceConsume"; int numRgUsageListenerCallbacks = 0; int numRgFillUsageCallbacks = 0; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImplTest.java index 7859066223e14..04edf67e275e7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImplTest.java @@ -32,7 +32,7 @@ public class ResourceQuotaCalculatorImplTest extends MockedPulsarServiceBaseTest @Override protected void setup() throws Exception { super.internalSetup(); - this.rqCalc = new ResourceQuotaCalculatorImpl(); + this.rqCalc = new ResourceQuotaCalculatorImpl(pulsar); } @AfterClass(alwaysRun = true) @@ -124,5 +124,16 @@ public void testNeedToReportLocalUsage() { Assert.assertTrue(rqCalc.needToReportLocalUsage(940, 1000, 94, 100, System.currentTimeMillis())); } + @Test + public void testUsedUsageLessThanConfigUsage() throws PulsarAdminException { + final long config = 100; + final long usedUsage = 100 / pulsar.getConfiguration().getResourceUsageTransportPublishIntervalInSecs() / 3; + final long[] allUsage = {usedUsage, usedUsage}; + for (long n : allUsage) { + long localQuota = rqCalc.computeLocalQuota(config, n, allUsage); + Assert.assertEquals(localQuota, config); + } + } + private ResourceQuotaCalculatorImpl rqCalc; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceUsageTransportManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceUsageTransportManagerTest.java index c79e0cc7910c9..22c49ae37f620 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceUsageTransportManagerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceUsageTransportManagerTest.java @@ -79,7 +79,7 @@ public void fillResourceUsage(ResourceUsage resourceUsage) { resourceUsage.setOwner(getID()); resourceUsage.setPublish().setMessagesPerPeriod(1000).setBytesPerPeriod(10001); resourceUsage.setStorage().setTotalBytes(500003); - + resourceUsage.setReplicationDispatch().setMessagesPerPeriod(2000).setBytesPerPeriod(4000); } }; @@ -98,6 +98,10 @@ public void acceptResourceUsage(String broker, ResourceUsage resourceUsage) { p.setBytesPerPeriod(resourceUsage.getPublish().getBytesPerPeriod()); p.setMessagesPerPeriod(resourceUsage.getPublish().getMessagesPerPeriod()); + p = recvdUsage.setReplicationDispatch(); + p.setBytesPerPeriod(resourceUsage.getReplicationDispatch().getBytesPerPeriod()); + p.setMessagesPerPeriod(resourceUsage.getReplicationDispatch().getMessagesPerPeriod()); + recvdUsage.setStorage().setTotalBytes(resourceUsage.getStorage().getTotalBytes()); } }; @@ -112,6 +116,8 @@ public void acceptResourceUsage(String broker, ResourceUsage resourceUsage) { assertNotNull(recvdUsage.getStorage()); assertEquals(recvdUsage.getPublish().getBytesPerPeriod(), 10001); assertEquals(recvdUsage.getStorage().getTotalBytes(), 500003); + assertEquals(recvdUsage.getReplicationDispatch().getBytesPerPeriod(), 4000); + assertEquals(recvdUsage.getReplicationDispatch().getMessagesPerPeriod(), 2000); } private void prepareData() throws PulsarServerException, PulsarAdminException, PulsarClientException { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java index 1e15bcf12c08d..38334ea387dbf 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java @@ -21,6 +21,7 @@ import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import static org.mockito.Mockito.withSettings; import static org.testng.Assert.assertEquals; @@ -40,7 +41,8 @@ public class AbstractTopicTest { public void beforeMethod() { BrokerService brokerService = mock(BrokerService.class); PulsarService pulsarService = mock(PulsarService.class); - ServiceConfiguration serviceConfiguration = mock(ServiceConfiguration.class); + ServiceConfiguration serviceConfiguration = spy(ServiceConfiguration.class); + serviceConfiguration.setClusterName("test-cluster"); BacklogQuotaManager backlogQuotaManager = mock(BacklogQuotaManager.class); subscription = mock(AbstractSubscription.class); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRateLimiterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRateLimiterTest.java index 2891bac26ce88..1d6d77e51f65f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRateLimiterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRateLimiterTest.java @@ -20,15 +20,19 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.AssertJUnit.assertFalse; import com.google.common.collect.Sets; import java.lang.reflect.Method; import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import lombok.Cleanup; +import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter; import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Consumer; @@ -36,6 +40,7 @@ import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.common.policies.data.DispatchRate; +import org.apache.pulsar.common.policies.data.ResourceGroup; import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -101,6 +106,7 @@ public void testReplicatorRateLimiterWithOnlyTopicLevel() throws Exception { // rate limiter disable by default assertFalse(getRateLimiter(topic).isPresent()); + assertFalse(getResourceGroupDispatchRateLimiter(topic).isPresent()); //set topic-level policy, which should take effect DispatchRate topicRate = DispatchRate.builder() @@ -122,6 +128,27 @@ public void testReplicatorRateLimiterWithOnlyTopicLevel() throws Exception { assertEquals(getRateLimiter(topic).get().getDispatchRateOnMsg(), -1); assertEquals(getRateLimiter(topic).get().getDispatchRateOnByte(), -1L); + + // ResourceGroupDispatchRateLimiter + String resourceGroupName = UUID.randomUUID().toString(); + ResourceGroup resourceGroup = new ResourceGroup(); + resourceGroup.setReplicationDispatchRateInBytes(10L); + resourceGroup.setReplicationDispatchRateInMsgs(20L); + admin1.resourcegroups().createResourceGroup(resourceGroupName, resourceGroup); + Awaitility.await().untilAsserted(() -> assertNotNull(admin1.resourcegroups() + .getResourceGroup(resourceGroupName))); + admin1.topicPolicies().setResourceGroup(topicName, resourceGroupName); + + Awaitility.await().untilAsserted(() -> { + Optional resourceGroupDispatchRateLimiterOpt = + getResourceGroupDispatchRateLimiter(topic); + assertTrue(resourceGroupDispatchRateLimiterOpt.isPresent()); + ResourceGroupDispatchLimiter resourceGroupDispatchLimiter = resourceGroupDispatchRateLimiterOpt.get(); + assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnByte(), + resourceGroup.getReplicationDispatchRateInBytes().longValue()); + assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnMsg(), + resourceGroup.getReplicationDispatchRateInMsgs().longValue()); + }); } @Test @@ -145,6 +172,7 @@ public void testReplicatorRateLimiterWithOnlyNamespaceLevel() throws Exception { // rate limiter disable by default assertFalse(getRateLimiter(topic).isPresent()); + assertFalse(getResourceGroupDispatchRateLimiter(topic).isPresent()); //set namespace-level policy, which should take effect DispatchRate topicRate = DispatchRate.builder() @@ -166,6 +194,27 @@ public void testReplicatorRateLimiterWithOnlyNamespaceLevel() throws Exception { assertEquals(getRateLimiter(topic).get().getDispatchRateOnMsg(), -1); assertEquals(getRateLimiter(topic).get().getDispatchRateOnByte(), -1L); + + // ResourceGroupDispatchRateLimiter + String resourceGroupName = UUID.randomUUID().toString(); + ResourceGroup resourceGroup = new ResourceGroup(); + resourceGroup.setReplicationDispatchRateInBytes(10L); + resourceGroup.setReplicationDispatchRateInMsgs(20L); + admin1.resourcegroups().createResourceGroup(resourceGroupName, resourceGroup); + Awaitility.await().untilAsserted(() -> assertNotNull(admin1.resourcegroups() + .getResourceGroup(resourceGroupName))); + admin1.namespaces().setNamespaceResourceGroup(namespace, resourceGroupName); + + Awaitility.await().untilAsserted(() -> { + Optional resourceGroupDispatchRateLimiterOpt = + getResourceGroupDispatchRateLimiter(topic); + assertTrue(resourceGroupDispatchRateLimiterOpt.isPresent()); + ResourceGroupDispatchLimiter resourceGroupDispatchLimiter = resourceGroupDispatchRateLimiterOpt.get(); + assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnByte(), + resourceGroup.getReplicationDispatchRateInBytes().longValue()); + assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnMsg(), + resourceGroup.getReplicationDispatchRateInMsgs().longValue()); + }); } @Test @@ -440,7 +489,7 @@ public void testReplicatorRateLimiterMessageNotReceivedAllMessages(DispatchRateT Consumer consumer = client2.newConsumer().topic(topicName) .subscriptionName("sub2-in-cluster2").messageListener((c1, msg) -> { - Assert.assertNotNull(msg, "Message cannot be null"); + assertNotNull(msg, "Message cannot be null"); String receivedMessage = new String(msg.getData()); log.debug("Received message [{}] in the listener", receivedMessage); totalReceived.incrementAndGet(); @@ -522,7 +571,7 @@ public void testReplicatorRateLimiterMessageReceivedAllMessages() throws Excepti Consumer consumer = client2.newConsumer().topic(topicName) .subscriptionName("sub2-in-cluster2").messageListener((c1, msg) -> { - Assert.assertNotNull(msg, "Message cannot be null"); + assertNotNull(msg, "Message cannot be null"); String receivedMessage = new String(msg.getData()); log.debug("Received message [{}] in the listener", receivedMessage); totalReceived.incrementAndGet(); @@ -554,6 +603,255 @@ public void testReplicatorRateLimiterMessageReceivedAllMessages() throws Excepti producer.close(); } + @Test + public void testResourceGroupReplicatorRateLimiter() throws Exception { + final String namespace = "pulsar/replicatormsg-" + System.currentTimeMillis(); + final String topicName = "persistent://" + namespace + "/" + UUID.randomUUID(); + + admin1.namespaces().createNamespace(namespace); + // 0. set 2 clusters, there will be 1 replicator in each topic + admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2")); + + // ResourceGroupDispatchRateLimiter + int messageRate = 100; + String resourceGroupName = UUID.randomUUID().toString(); + ResourceGroup resourceGroup = new ResourceGroup(); + resourceGroup.setReplicationDispatchRateInMsgs((long) messageRate); + admin1.resourcegroups().createResourceGroup(resourceGroupName, resourceGroup); + Awaitility.await().untilAsserted(() -> assertNotNull(admin1.resourcegroups() + .getResourceGroup(resourceGroupName))); + admin1.namespaces().setNamespaceResourceGroup(namespace, resourceGroupName); + + @Cleanup + PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS) + .build(); + + @Cleanup + Producer producer = client1.newProducer().topic(topicName) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + + @Cleanup + PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()).statsInterval(0, TimeUnit.SECONDS) + .build(); + final AtomicInteger totalReceived = new AtomicInteger(0); + + @Cleanup + Consumer consumer = client2.newConsumer().topic(topicName).subscriptionName("sub2-in-cluster2").messageListener((c1, msg) -> { + Assert.assertNotNull(msg, "Message cannot be null"); + String receivedMessage = new String(msg.getData()); + log.debug("Received message [{}] in the listener", receivedMessage); + totalReceived.incrementAndGet(); + }).subscribe(); + + int numMessages = 500; + for (int i = 0; i < numMessages; i++) { + producer.send(new byte[80]); + } + + Assert.assertTrue(totalReceived.get() < messageRate * 2); + } + + @Test + public void testLoadResourceGroupReplicatorRateLimiter() throws Exception { + final String namespace = "pulsar/replicatormsg-" + System.currentTimeMillis(); + final String topicName1 = "persistent://" + namespace + "/" + UUID.randomUUID(); + final String topicName2= "persistent://" + namespace + "/" + UUID.randomUUID(); + + admin1.namespaces().createNamespace(namespace); + // 0. set 2 clusters, there will be 1 replicator in each topic + admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2")); + + // ResourceGroupDispatchRateLimiter + long messageRateOnNamespace = 100; + long byteRateOnNamespace = 500000; + String resourceGroupNameOnNamespace = UUID.randomUUID().toString(); + ResourceGroup resourceGroupOnNamespace = new ResourceGroup(); + resourceGroupOnNamespace.setReplicationDispatchRateInMsgs(messageRateOnNamespace); + resourceGroupOnNamespace.setReplicationDispatchRateInBytes(byteRateOnNamespace); + admin1.resourcegroups().createResourceGroup(resourceGroupNameOnNamespace, resourceGroupOnNamespace); + Awaitility.await().untilAsserted(() -> assertNotNull(admin1.resourcegroups() + .getResourceGroup(resourceGroupNameOnNamespace))); + admin1.namespaces().setNamespaceResourceGroup(namespace, resourceGroupNameOnNamespace); + + @Cleanup + PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS) + .build(); + + @Cleanup + Producer producer = client1.newProducer().topic(topicName1) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + producer.send(new byte[1]); + @Cleanup + Producer producer2 = client1.newProducer().topic(topicName2) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + producer2.send(new byte[1]); + + CompletableFuture> topic1IfExists = pulsar1.getBrokerService().getTopicIfExists(topicName1); + assertThat(topic1IfExists).succeedsWithin(3, TimeUnit.SECONDS); + Optional topic1Optional = topic1IfExists.get(); + assertTrue(topic1Optional.isPresent()); + PersistentTopic persistentTopic1 = (PersistentTopic) topic1Optional.get(); + + Replicator topic1WithR2Replicator = persistentTopic1.getReplicators().get("r2"); + assertNotNull(topic1WithR2Replicator); + Optional topic1ResourceGroupDispatchRateLimiter = + topic1WithR2Replicator.getResourceGroupDispatchRateLimiter(); + assertTrue(topic1ResourceGroupDispatchRateLimiter.isPresent()); + assertEquals(topic1ResourceGroupDispatchRateLimiter.get().getDispatchRateOnMsg(), messageRateOnNamespace); + assertEquals(topic1ResourceGroupDispatchRateLimiter.get().getDispatchRateOnByte(), byteRateOnNamespace); + + CompletableFuture> topic2IfExists = pulsar1.getBrokerService().getTopicIfExists(topicName2); + assertThat(topic2IfExists).succeedsWithin(3, TimeUnit.SECONDS); + Optional topic2Optional = topic1IfExists.get(); + assertTrue(topic2Optional.isPresent()); + PersistentTopic persistentTopic2 = (PersistentTopic) topic1Optional.get(); + + Replicator topic2WithR2Replicator = persistentTopic2.getReplicators().get("r2"); + assertNotNull(topic2WithR2Replicator); + Optional topic2ResourceGroupDispatchRateLimiter = + topic2WithR2Replicator.getResourceGroupDispatchRateLimiter(); + assertTrue(topic2ResourceGroupDispatchRateLimiter.isPresent()); + assertEquals(topic2ResourceGroupDispatchRateLimiter.get().getDispatchRateOnMsg(), messageRateOnNamespace); + assertEquals(topic2ResourceGroupDispatchRateLimiter.get().getDispatchRateOnByte(), byteRateOnNamespace); + + // Ensure to use the same limiter. + assertEquals(topic1ResourceGroupDispatchRateLimiter.get(), topic2ResourceGroupDispatchRateLimiter.get()); + + // Set up rate limiter for r1 -> r2 channel. + long messageRateOnNamespaceBetweenR1AndR2 = 1000; + long byteRateOnNamespaceBetweenR1AndR2 = 5000000; + admin1.resourcegroups().setReplicatorDispatchRate(resourceGroupNameOnNamespace, "r2", + DispatchRate.builder() + .dispatchThrottlingRateInByte(byteRateOnNamespaceBetweenR1AndR2) + .dispatchThrottlingRateInMsg((int) messageRateOnNamespaceBetweenR1AndR2) + .build()); + Awaitility.await().untilAsserted(() -> { + Optional rateLimiter1 = + topic1WithR2Replicator.getResourceGroupDispatchRateLimiter(); + assertTrue(rateLimiter1.isPresent()); + assertEquals(rateLimiter1.get().getDispatchRateOnMsg(), messageRateOnNamespaceBetweenR1AndR2); + assertEquals(rateLimiter1.get().getDispatchRateOnByte(), byteRateOnNamespaceBetweenR1AndR2); + Optional rateLimiter2 = + topic2WithR2Replicator.getResourceGroupDispatchRateLimiter(); + assertTrue(rateLimiter2.isPresent()); + assertEquals(rateLimiter2.get().getDispatchRateOnMsg(), messageRateOnNamespaceBetweenR1AndR2); + assertEquals(rateLimiter2.get().getDispatchRateOnByte(), byteRateOnNamespaceBetweenR1AndR2); + + assertEquals(rateLimiter1.get(), rateLimiter2.get()); + }); + + // Remove rate limiter for r1 -> r2 channel, and then use the default rate limiter. + admin1.resourcegroups().removeReplicatorDispatchRate(resourceGroupNameOnNamespace, "r2"); + Awaitility.await().untilAsserted(() -> { + Optional rateLimiter1 = + topic1WithR2Replicator.getResourceGroupDispatchRateLimiter(); + assertTrue(rateLimiter1.isPresent()); + assertEquals(rateLimiter1.get().getDispatchRateOnMsg(), messageRateOnNamespace); + assertEquals(rateLimiter1.get().getDispatchRateOnByte(), byteRateOnNamespace); + Optional rateLimiter2 = + topic2WithR2Replicator.getResourceGroupDispatchRateLimiter(); + assertTrue(rateLimiter2.isPresent()); + assertEquals(rateLimiter2.get().getDispatchRateOnMsg(), messageRateOnNamespace); + assertEquals(rateLimiter2.get().getDispatchRateOnByte(), byteRateOnNamespace); + + assertEquals(rateLimiter1.get(), rateLimiter2.get()); + }); + + long messageRateOnTopic = 1001; + long byteRateOnTopic = 5000000; + String resourceGroupNameOnTopic = UUID.randomUUID().toString(); + ResourceGroup resourceGroupOnTopic = new ResourceGroup(); + resourceGroupOnTopic.setReplicationDispatchRateInMsgs(messageRateOnTopic); + resourceGroupOnTopic.setReplicationDispatchRateInBytes(byteRateOnTopic); + admin1.resourcegroups().createResourceGroup(resourceGroupNameOnTopic, resourceGroupOnTopic); + Awaitility.await().untilAsserted(() -> assertNotNull(admin1.resourcegroups() + .getResourceGroup(resourceGroupNameOnTopic))); + admin1.topicPolicies().setResourceGroup(topicName1, resourceGroupNameOnTopic); + admin1.topicPolicies().setResourceGroup(topicName2, resourceGroupNameOnTopic); + Awaitility.await().untilAsserted(() ->{ + assertEquals(admin1.topicPolicies() + .getResourceGroup(topicName1, false), resourceGroupNameOnTopic); + assertEquals(admin1.topicPolicies() + .getResourceGroup(topicName2, false), resourceGroupNameOnTopic); + }); + Awaitility.await().untilAsserted(() -> { + Optional rateLimiter1 = + topic1WithR2Replicator.getResourceGroupDispatchRateLimiter(); + assertTrue(rateLimiter1.isPresent()); + assertEquals(rateLimiter1.get().getDispatchRateOnMsg(), messageRateOnTopic); + assertEquals(rateLimiter1.get().getDispatchRateOnByte(), byteRateOnTopic); + Optional rateLimiter2 = + topic2WithR2Replicator.getResourceGroupDispatchRateLimiter(); + assertTrue(rateLimiter2.isPresent()); + assertEquals(rateLimiter2.get().getDispatchRateOnMsg(), messageRateOnTopic); + assertEquals(rateLimiter2.get().getDispatchRateOnByte(), byteRateOnTopic); + + assertEquals(rateLimiter1.get(), rateLimiter2.get()); + }); + + // Set up rate limiter for r1 -> r2 channel on the topic. + long messageRateOnTopicBetweenR1AndR2 = 1002; + long byteRateOnTopicBetweenR1AndR2 = 5000001; + admin1.resourcegroups().setReplicatorDispatchRate(resourceGroupNameOnTopic, "r2", + DispatchRate.builder() + .dispatchThrottlingRateInByte(byteRateOnTopicBetweenR1AndR2) + .dispatchThrottlingRateInMsg((int) messageRateOnTopicBetweenR1AndR2) + .build()); + Awaitility.await().untilAsserted(() -> { + Optional rateLimiter1 = + topic1WithR2Replicator.getResourceGroupDispatchRateLimiter(); + assertTrue(rateLimiter1.isPresent()); + assertEquals(rateLimiter1.get().getDispatchRateOnMsg(), messageRateOnTopicBetweenR1AndR2); + assertEquals(rateLimiter1.get().getDispatchRateOnByte(), byteRateOnTopicBetweenR1AndR2); + Optional rateLimiter2 = + topic2WithR2Replicator.getResourceGroupDispatchRateLimiter(); + assertTrue(rateLimiter2.isPresent()); + assertEquals(rateLimiter2.get().getDispatchRateOnMsg(), messageRateOnTopicBetweenR1AndR2); + assertEquals(rateLimiter2.get().getDispatchRateOnByte(), byteRateOnTopicBetweenR1AndR2); + + assertEquals(rateLimiter1.get(), rateLimiter2.get()); + }); + + // Remove rate limiter for r1 -> r2 channel on the topic, and then use the default rate limiter. + admin1.resourcegroups().removeReplicatorDispatchRate(resourceGroupNameOnTopic, "r2"); + Awaitility.await().untilAsserted(() -> { + Optional rateLimiter1 = + topic1WithR2Replicator.getResourceGroupDispatchRateLimiter(); + assertTrue(rateLimiter1.isPresent()); + assertEquals(rateLimiter1.get().getDispatchRateOnMsg(), messageRateOnTopic); + assertEquals(rateLimiter1.get().getDispatchRateOnByte(), byteRateOnTopic); + Optional rateLimiter2 = + topic2WithR2Replicator.getResourceGroupDispatchRateLimiter(); + assertTrue(rateLimiter2.isPresent()); + assertEquals(rateLimiter2.get().getDispatchRateOnMsg(), messageRateOnTopic); + assertEquals(rateLimiter2.get().getDispatchRateOnByte(), byteRateOnTopic); + + assertEquals(rateLimiter1.get(), rateLimiter2.get()); + }); + + admin1.topicPolicies().removeResourceGroup(topicName1); + Awaitility.await().untilAsserted(() -> { + Optional rateLimiter1 = + topic1WithR2Replicator.getResourceGroupDispatchRateLimiter(); + assertTrue(rateLimiter1.isPresent()); + assertEquals(rateLimiter1.get().getDispatchRateOnMsg(), messageRateOnNamespace); + assertEquals(rateLimiter1.get().getDispatchRateOnByte(), byteRateOnNamespace); + Optional rateLimiter2 = + topic2WithR2Replicator.getResourceGroupDispatchRateLimiter(); + assertTrue(rateLimiter2.isPresent()); + assertEquals(rateLimiter2.get().getDispatchRateOnMsg(), messageRateOnNamespace); + assertEquals(rateLimiter2.get().getDispatchRateOnByte(), byteRateOnNamespace); + + assertEquals(rateLimiter1.get(), rateLimiter2.get()); + }); + } + @Test public void testReplicatorRateLimiterByBytes() throws Exception { final String namespace = "pulsar/replicatormsg-" + System.currentTimeMillis(); @@ -594,7 +892,7 @@ public void testReplicatorRateLimiterByBytes() throws Exception { @Cleanup Consumer ignored = client2.newConsumer().topic(topicName).subscriptionName("sub2-in-cluster2") .messageListener((c1, msg) -> { - Assert.assertNotNull(msg, "Message cannot be null"); + assertNotNull(msg, "Message cannot be null"); String receivedMessage = new String(msg.getData()); log.debug("Received message [{}] in the listener", receivedMessage); totalReceived.incrementAndGet(); @@ -617,5 +915,165 @@ private static Optional getRateLimiter(PersistentTopic topi return topic.getReplicators().values().stream().findFirst().map(Replicator::getRateLimiter).orElseThrow(); } + private static Optional getResourceGroupDispatchRateLimiter(PersistentTopic topic) { + return topic.getReplicators().values().stream().findFirst().map(Replicator::getResourceGroupDispatchRateLimiter) + .orElseThrow(); + } + + @Test + public void testLoadReplicatorDispatchRateLimiterByTopicPolicies() throws Exception { + final String namespace = "pulsar/replicator-dispatch-rate-" + System.currentTimeMillis(); + final String topicName = "persistent://" + namespace + "/" + System.currentTimeMillis(); + + admin1.namespaces().createNamespace(namespace); + admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2")); + + final int byteRate = 400; + DispatchRate dispatchRate = DispatchRate.builder() + .dispatchThrottlingRateInMsg(-1) + .dispatchThrottlingRateInByte(byteRate) + .ratePeriodInSecond(1) + .build(); + admin1.namespaces().setReplicatorDispatchRate(namespace, dispatchRate); + + admin1.topics().createNonPartitionedTopic(topicName); + + Optional topicOptional = pulsar1.getBrokerService().getTopicIfExists(topicName).get(); + assertTrue(topicOptional.isPresent()); + + PersistentTopic topic = (PersistentTopic) topicOptional.get(); + Awaitility.await() + .untilAsserted(() -> { + Optional rateLimiter = getRateLimiter(topic); + assertTrue(rateLimiter.isPresent()); + assertEquals(rateLimiter.get().getDispatchRateOnByte(), byteRate); + }); + + // r1 -> r2: limits this replication channel by default rate limit on the namespace level. + long dispatchThrottlingRateInMsgOnNs = 1000; + long dispatchThrottlingRateInByteOnNs = 3000; + DispatchRate dispatchRateOnNamespace = DispatchRate.builder() + .dispatchThrottlingRateInMsg((int) dispatchThrottlingRateInMsgOnNs) + .dispatchThrottlingRateInByte(dispatchThrottlingRateInByteOnNs) + .ratePeriodInSecond(1) + .build(); + admin1.namespaces().setReplicatorDispatchRate(namespace, dispatchRateOnNamespace); + Awaitility.await() + .untilAsserted(() -> { + Optional rateLimiter = getRateLimiter(topic); + assertTrue(rateLimiter.isPresent()); + assertEquals(rateLimiter.get().getDispatchRateOnByte(), dispatchThrottlingRateInByteOnNs); + assertEquals(rateLimiter.get().getDispatchRateOnMsg(), dispatchThrottlingRateInMsgOnNs); + }); + + // r1 -> r2: limits this replication channel by the specific cluster on the namespace level. + long dispatchThrottlingRateInMsgWhenR1ToR2OnNs = 3000; + long dispatchThrottlingRateInByteWhenR1ToR2OnNs = 2000; + DispatchRate dispatchThrottlingRateInMsgBetweenR1AndR2OnNs = DispatchRate.builder() + .dispatchThrottlingRateInMsg((int) dispatchThrottlingRateInMsgWhenR1ToR2OnNs) + .dispatchThrottlingRateInByte(dispatchThrottlingRateInByteWhenR1ToR2OnNs) + .ratePeriodInSecond(1) + .build(); + admin1.namespaces().setReplicatorDispatchRate(namespace, "r2", dispatchThrottlingRateInMsgBetweenR1AndR2OnNs); + Awaitility.await() + .untilAsserted(() -> { + Optional rateLimiter = getRateLimiter(topic); + assertTrue(rateLimiter.isPresent()); + assertEquals(rateLimiter.get().getDispatchRateOnByte(), dispatchThrottlingRateInByteWhenR1ToR2OnNs); + assertEquals(rateLimiter.get().getDispatchRateOnMsg(), dispatchThrottlingRateInMsgWhenR1ToR2OnNs); + }); + + // r1 -> r2: limits this replication channel by default dispatch rate on the topic level. + long defaultDispatchThrottlingRateInMsgOnTopic = 30000; + long defaultDispatchThrottlingRateInByteOnTopic = 20000; + DispatchRate dispatchRateOnTopic = DispatchRate.builder() + .dispatchThrottlingRateInMsg((int) defaultDispatchThrottlingRateInMsgOnTopic) + .dispatchThrottlingRateInByte(defaultDispatchThrottlingRateInByteOnTopic) + .ratePeriodInSecond(1) + .build(); + + admin1.topicPolicies().setReplicatorDispatchRate(topicName, dispatchRateOnTopic); + Awaitility.await() + .untilAsserted(() -> { + Optional rateLimiter = getRateLimiter(topic); + assertTrue(rateLimiter.isPresent()); + assertEquals(rateLimiter.get().getDispatchRateOnByte(), + defaultDispatchThrottlingRateInByteOnTopic); + assertEquals(rateLimiter.get().getDispatchRateOnMsg(), + defaultDispatchThrottlingRateInMsgOnTopic); + }); + + // r1 -> r2: limits this replication channel by the specific cluster on the topic level. + long dispatchThrottlingRateInMsgBetweenR1AndR2OnTopic = 50000; + long dispatchThrottlingRateInByteBetweenR1AndR2OnTopic = 40000; + DispatchRate dispatchRateBetweenR1AndR2OnTopic = DispatchRate.builder() + .dispatchThrottlingRateInMsg((int) dispatchThrottlingRateInMsgBetweenR1AndR2OnTopic) + .dispatchThrottlingRateInByte(dispatchThrottlingRateInByteBetweenR1AndR2OnTopic) + .ratePeriodInSecond(1) + .build(); + + admin1.topicPolicies().setReplicatorDispatchRate(topicName, "r2", dispatchRateBetweenR1AndR2OnTopic); + Awaitility.await() + .untilAsserted(() -> { + Optional rateLimiter = getRateLimiter(topic); + assertTrue(rateLimiter.isPresent()); + assertEquals(rateLimiter.get().getDispatchRateOnByte(), + dispatchThrottlingRateInByteBetweenR1AndR2OnTopic); + assertEquals(rateLimiter.get().getDispatchRateOnMsg(), + dispatchThrottlingRateInMsgBetweenR1AndR2OnTopic); + }); + + // r1 -> r2: removes the specific cluster rate limit from the topic level, and then will use the default rate + // limit from the topic level. + admin1.topicPolicies().removeReplicatorDispatchRate(topicName, "r2"); + Awaitility.await() + .untilAsserted(() -> { + Optional rateLimiter = getRateLimiter(topic); + assertTrue(rateLimiter.isPresent()); + assertEquals(rateLimiter.get().getDispatchRateOnByte(), + defaultDispatchThrottlingRateInByteOnTopic); + assertEquals(rateLimiter.get().getDispatchRateOnMsg(), + defaultDispatchThrottlingRateInMsgOnTopic); + }); + + // r1 -> r2: removes the default rate limit from the topic level, and then will use the specific cluster rate + // limit from the namespace level. + admin1.topicPolicies().removeReplicatorDispatchRate(topicName); + Awaitility.await() + .untilAsserted(() -> { + Optional rateLimiter = getRateLimiter(topic); + assertTrue(rateLimiter.isPresent()); + assertEquals(rateLimiter.get().getDispatchRateOnByte(), + dispatchThrottlingRateInByteWhenR1ToR2OnNs); + assertEquals(rateLimiter.get().getDispatchRateOnMsg(), + dispatchThrottlingRateInMsgWhenR1ToR2OnNs); + }); + + // r1 -> r2: removes the specific cluster rate limit from the namespace level, and then will use the default + // rate limit from the namespace level. + admin1.namespaces().removeReplicatorDispatchRate(namespace, "r2"); + Awaitility.await() + .untilAsserted(() -> { + Optional rateLimiter = getRateLimiter(topic); + assertTrue(rateLimiter.isPresent()); + assertEquals(rateLimiter.get().getDispatchRateOnByte(), + dispatchThrottlingRateInByteOnNs); + assertEquals(rateLimiter.get().getDispatchRateOnMsg(), + dispatchThrottlingRateInMsgOnNs); + }); + + // r1 -> r2: removes the default cluster rate limit from the namespace level, and then will use the + // rate limit from the broker level. + admin1.namespaces().removeReplicatorDispatchRate(namespace); + Awaitility.await() + .untilAsserted(() -> { + Optional rateLimiter = getRateLimiter(topic); + assertTrue(rateLimiter.isPresent()); + assertEquals(rateLimiter.get().getDispatchRateOnByte(), -1); + assertEquals(rateLimiter.get().getDispatchRateOnMsg(), -1); + }); + } + + private static final Logger log = LoggerFactory.getLogger(ReplicatorRateLimiterTest.class); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index ee292750263d8..a6a0417884f40 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -1589,6 +1589,7 @@ public void testTBRecoverChangeStateError() throws InterruptedException, Timeout ServiceConfiguration serviceConfiguration = mock(ServiceConfiguration.class); when(serviceConfiguration.isEnableReplicatedSubscriptions()).thenReturn(false); when(serviceConfiguration.isTransactionCoordinatorEnabled()).thenReturn(true); + when(serviceConfiguration.getClusterName()).thenReturn("test-cluster"); // Mock executorProvider. ExecutorProvider executorProvider = mock(ExecutorProvider.class); when(executorProvider.getExecutor(any(Object.class))).thenReturn(executorServiceRecover); diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java index 361ee60e180a8..29105bb0af751 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java @@ -2469,6 +2469,21 @@ CompletableFuture getTopicHashPositionsAsync(String namespac */ void setReplicatorDispatchRate(String namespace, DispatchRate dispatchRate) throws PulsarAdminException; + /** + * Set replicator-message-dispatch-rate. + *

+ * Replicators under this namespace can dispatch this many messages per second. + * + * @param namespace + * @param cluster The cluster for which to set the replicator dispatch rate. + * @param dispatchRate + * number of messages per second + * @throws PulsarAdminException + * Unexpected error + */ + void setReplicatorDispatchRate(String namespace, String cluster, DispatchRate dispatchRate) + throws PulsarAdminException; + /** * Set replicator-message-dispatch-rate asynchronously. *

@@ -2480,6 +2495,18 @@ CompletableFuture getTopicHashPositionsAsync(String namespac */ CompletableFuture setReplicatorDispatchRateAsync(String namespace, DispatchRate dispatchRate); + /** + * Set replicator-message-dispatch-rate asynchronously. + *

+ * Replicators under this namespace can dispatch this many messages per second. + * + * @param namespace + * @param cluster The cluster for which to set the replicator dispatch rate. + * @param dispatchRate + * number of messages per second + */ + CompletableFuture setReplicatorDispatchRateAsync(String namespace, String cluster, DispatchRate dispatchRate); + /** * Remove replicator-message-dispatch-rate. * @@ -2489,6 +2516,16 @@ CompletableFuture getTopicHashPositionsAsync(String namespac */ void removeReplicatorDispatchRate(String namespace) throws PulsarAdminException; + /** + * Remove replicator-message-dispatch-rate. + * + * @param namespace + * @param cluster The cluster for which to remove the replicator dispatch rate. + * @throws PulsarAdminException + * Unexpected error + */ + void removeReplicatorDispatchRate(String namespace, String cluster) throws PulsarAdminException; + /** * Set replicator-message-dispatch-rate asynchronously. * @@ -2496,6 +2533,14 @@ CompletableFuture getTopicHashPositionsAsync(String namespac */ CompletableFuture removeReplicatorDispatchRateAsync(String namespace); + /** + * Set replicator-message-dispatch-rate asynchronously. + * + * @param namespace + * @param cluster The cluster for which to remove the replicator dispatch rate. + */ + CompletableFuture removeReplicatorDispatchRateAsync(String namespace, String cluster); + /** * Get replicator-message-dispatch-rate. *

@@ -2509,6 +2554,20 @@ CompletableFuture getTopicHashPositionsAsync(String namespac */ DispatchRate getReplicatorDispatchRate(String namespace) throws PulsarAdminException; + /** + * Get replicator-message-dispatch-rate. + *

+ * Replicators under this namespace can dispatch this many messages per second. + * + * @param namespace + * @param cluster The cluster for which to get the replicator dispatch rate. + * @returns DispatchRate + * number of messages per second + * @throws PulsarAdminException + * Unexpected error + */ + DispatchRate getReplicatorDispatchRate(String namespace, String cluster) throws PulsarAdminException; + /** * Get replicator-message-dispatch-rate asynchronously. *

@@ -2520,6 +2579,18 @@ CompletableFuture getTopicHashPositionsAsync(String namespac */ CompletableFuture getReplicatorDispatchRateAsync(String namespace); + /** + * Get replicator-message-dispatch-rate asynchronously. + *

+ * Replicators under this namespace can dispatch this many messages per second. + * + * @param namespace + * @param cluster The cluster for which to get the dispatch rate. + * @returns DispatchRate + * number of messages per second + */ + CompletableFuture getReplicatorDispatchRateAsync(String namespace, String cluster); + /** * Clear backlog for all topics on a namespace. * diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/ResourceGroups.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/ResourceGroups.java index 843976eb13d42..e807fa24b96d2 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/ResourceGroups.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/ResourceGroups.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.concurrent.CompletableFuture; +import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.ResourceGroup; /** @@ -75,6 +76,8 @@ public interface ResourceGroups { * "PublishRateInBytes" : "value", * "DispatchRateInMsgs" : "value", * "DispatchRateInBytes" : "value" + * "ReplicationDispatchRateInMsgs" : "value" + * "ReplicationDispatchRateInBytes" : "value" * * * @@ -101,6 +104,8 @@ public interface ResourceGroups { * "PublishRateInBytes" : "value", * "DispatchRateInMsgs" : "value", * "DspatchRateInBytes" : "value" + * "ReplicationDispatchRateInMsgs" : "value" + * "ReplicationDispatchRateInBytes" : "value" * * * @@ -183,4 +188,53 @@ public interface ResourceGroups { CompletableFuture deleteResourceGroupAsync(String resourcegroup); + /** + * Set replicator message dispatch rate from a resource group. + * + * @param resourcegroup Resourcegroup name. + * @param cluster The remote cluster for which to set the replicator dispatch rate. + */ + void setReplicatorDispatchRate(String resourcegroup, String cluster, DispatchRate dispatchRate) + throws PulsarAdminException; + + /** + * Set replicator message dispatch rate from a resource group asynchronously. + * + * @param resourcegroup Resourcegroup name. + * @param cluster The remote cluster for which to set the replicator dispatch rate. + */ + CompletableFuture setReplicatorDispatchRateAsync(String resourcegroup, String cluster, + DispatchRate dispatchRate); + + /** + * Remove replicator message dispatch rate from a resource group. + * + * @param resourcegroup Resourcegroup name. + * @param cluster The remote cluster for which to remove the replicator dispatch rate. + */ + void removeReplicatorDispatchRate(String resourcegroup, String cluster) throws PulsarAdminException; + + /** + * Remove replicator message dispatch rate from a resource group asynchronously. + * + * @param resourcegroup Resourcegroup name. + * @param cluster The remote cluster for which to remove the replicator dispatch rate. + */ + CompletableFuture removeReplicatorDispatchRateAsync(String resourcegroup, String cluster); + + /** + * Get replicator message dispatch rate for a resource group. + * + * @param resourcegroup Resourcegroup name. + * @param cluster The remote cluster for which to get the replicator dispatch rate. + */ + DispatchRate getReplicatorDispatchRate(String resourcegroup, String cluster) throws PulsarAdminException; + + /** + * Get replicator message dispatch rate for a resource group asynchronously. + * + * @param resourcegroup Resourcegroup name. + * @param cluster The remote cluster for which to get the replicator dispatch rate. + */ + CompletableFuture getReplicatorDispatchRateAsync(String resourcegroup, String cluster); } \ No newline at end of file diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/TopicPolicies.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/TopicPolicies.java index 721012db061ca..77c50b8241a1c 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/TopicPolicies.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/TopicPolicies.java @@ -971,6 +971,20 @@ CompletableFuture getSubscriptionDispatchRateAsync(String topic, S */ void setReplicatorDispatchRate(String topic, DispatchRate dispatchRate) throws PulsarAdminException; + /** + * Set replicatorDispatchRate for the topic. + *

+ * Replicator dispatch rate under this topic can dispatch this many messages per second + * + * @param topic + * @param cluster The cluster for which set the replicator dispatch rate. + * @param dispatchRate + * number of messages per second + * @throws PulsarAdminException + * Unexpected error + */ + void setReplicatorDispatchRate(String topic, String cluster, DispatchRate dispatchRate) throws PulsarAdminException; + /** * Set replicatorDispatchRate for the topic asynchronously. *

@@ -982,6 +996,17 @@ CompletableFuture getSubscriptionDispatchRateAsync(String topic, S */ CompletableFuture setReplicatorDispatchRateAsync(String topic, DispatchRate dispatchRate); + /** + * Set replicatorDispatchRate for the topic asynchronously. + *

+ * Replicator dispatch rate under this topic can dispatch this many messages per second. + * + * @param topic + * @param cluster The cluster for which set the replicator dispatch rate. + * @param dispatchRate number of messages per second + */ + CompletableFuture setReplicatorDispatchRateAsync(String topic, String cluster, DispatchRate dispatchRate); + /** * Get replicatorDispatchRate for the topic. *

@@ -1015,6 +1040,16 @@ CompletableFuture getSubscriptionDispatchRateAsync(String topic, S */ DispatchRate getReplicatorDispatchRate(String topic, boolean applied) throws PulsarAdminException; + /** + * Get applied replicatorDispatchRate for the topic. + * @param topic + * @param cluster The cluster for which get the replicator dispatch rate. + * @param applied + * @return + * @throws PulsarAdminException + */ + DispatchRate getReplicatorDispatchRate(String topic, String cluster, boolean applied) throws PulsarAdminException; + /** * Get applied replicatorDispatchRate asynchronously. * @param topic @@ -1023,6 +1058,15 @@ CompletableFuture getSubscriptionDispatchRateAsync(String topic, S */ CompletableFuture getReplicatorDispatchRateAsync(String topic, boolean applied); + /** + * Get applied replicatorDispatchRate asynchronously. + * @param topic + * @param cluster The cluster for which get the replicator dispatch rate. + * @param applied + * @return + */ + CompletableFuture getReplicatorDispatchRateAsync(String topic, String cluster, boolean applied); + /** * Remove replicatorDispatchRate for a topic. * @param topic @@ -1032,6 +1076,16 @@ CompletableFuture getSubscriptionDispatchRateAsync(String topic, S */ void removeReplicatorDispatchRate(String topic) throws PulsarAdminException; + /** + * Remove replicatorDispatchRate for a topic. + * @param topic + * Topic name + * @param cluster The cluster for which remove the replicator dispatch rate. + * @throws PulsarAdminException + * Unexpected error + */ + void removeReplicatorDispatchRate(String topic, String cluster) throws PulsarAdminException; + /** * Remove replicatorDispatchRate for a topic asynchronously. * @param topic @@ -1039,6 +1093,14 @@ CompletableFuture getSubscriptionDispatchRateAsync(String topic, S */ CompletableFuture removeReplicatorDispatchRateAsync(String topic); + /** + * Remove replicatorDispatchRate for a topic asynchronously. + * @param topic + * Topic name + * @param cluster The cluster for which remove the replicator dispatch rate. + */ + CompletableFuture removeReplicatorDispatchRateAsync(String topic, String cluster); + /** * Get the compactionThreshold for a topic. The maximum number of bytes * can have before compaction is triggered. 0 disables. diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ResourceGroup.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ResourceGroup.java index bd47427e9aa82..dc7d0e74361ee 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ResourceGroup.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ResourceGroup.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.common.policies.data; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import lombok.Data; @Data @@ -26,4 +28,9 @@ public class ResourceGroup { private Long publishRateInBytes; private Integer dispatchRateInMsgs; private Long dispatchRateInBytes; + + private Long replicationDispatchRateInMsgs; + private Long replicationDispatchRateInBytes; + + private Map replicatorDispatchRate = new ConcurrentHashMap<>(); } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java index 233d6a6328cc3..de9a37ae3b7ab 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java @@ -1053,8 +1053,20 @@ public void setReplicatorDispatchRate(String namespace, DispatchRate dispatchRat @Override public CompletableFuture setReplicatorDispatchRateAsync(String namespace, DispatchRate dispatchRate) { + return setReplicatorDispatchRateAsync(namespace, null, dispatchRate); + } + + @Override + public void setReplicatorDispatchRate(String namespace, String cluster, DispatchRate dispatchRate) + throws PulsarAdminException { + sync(() -> setReplicatorDispatchRateAsync(namespace, cluster, dispatchRate)); + } + + @Override + public CompletableFuture setReplicatorDispatchRateAsync(String namespace, String cluster, + DispatchRate dispatchRate) { NamespaceName ns = NamespaceName.get(namespace); - WebTarget path = namespacePath(ns, "replicatorDispatchRate"); + WebTarget path = namespacePath(ns, "replicatorDispatchRate").queryParam("cluster", cluster); return asyncPostRequest(path, Entity.entity(dispatchRate, MediaType.APPLICATION_JSON)); } @@ -1065,8 +1077,18 @@ public void removeReplicatorDispatchRate(String namespace) throws PulsarAdminExc @Override public CompletableFuture removeReplicatorDispatchRateAsync(String namespace) { + return removeReplicatorDispatchRateAsync(namespace, null); + } + + @Override + public void removeReplicatorDispatchRate(String namespace, String cluster) throws PulsarAdminException { + sync(() -> removeReplicatorDispatchRateAsync(namespace, cluster)); + } + + @Override + public CompletableFuture removeReplicatorDispatchRateAsync(String namespace, String cluster) { NamespaceName ns = NamespaceName.get(namespace); - WebTarget path = namespacePath(ns, "replicatorDispatchRate"); + WebTarget path = namespacePath(ns, "replicatorDispatchRate").queryParam("cluster", cluster); return asyncDeleteRequest(path); } @@ -1075,9 +1097,21 @@ public DispatchRate getReplicatorDispatchRate(String namespace) throws PulsarAdm return sync(() -> getReplicatorDispatchRateAsync(namespace)); } + @Override + public DispatchRate getReplicatorDispatchRate(String namespace, String cluster) throws PulsarAdminException { + return sync(()-> getReplicatorDispatchRateAsync(namespace, cluster)); + } + @Override public CompletableFuture getReplicatorDispatchRateAsync(String namespace) { - return asyncGetNamespaceParts(new FutureCallback(){}, namespace, "replicatorDispatchRate"); + return getReplicatorDispatchRateAsync(namespace, null); + } + + @Override + public CompletableFuture getReplicatorDispatchRateAsync(String namespace, String cluster) { + NamespaceName ns = NamespaceName.get(namespace); + WebTarget path = namespacePath(ns, "replicatorDispatchRate").queryParam("cluster", cluster); + return asyncGetRequest(path, DispatchRate.class); } @Override diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ResourceGroupsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ResourceGroupsImpl.java index 4e7230eebd980..555b570cdb3e9 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ResourceGroupsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ResourceGroupsImpl.java @@ -26,6 +26,7 @@ import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.admin.ResourceGroups; import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.ResourceGroup; @@ -90,4 +91,42 @@ public CompletableFuture deleteResourceGroupAsync(String name) { WebTarget path = adminResourceGroups.path(name); return asyncDeleteRequest(path); } + + @Override + public void setReplicatorDispatchRate(String resourcegroup, String cluster, DispatchRate dispatchRate) + throws PulsarAdminException { + sync(() -> setReplicatorDispatchRateAsync(resourcegroup, cluster, dispatchRate)); + } + + @Override + public CompletableFuture setReplicatorDispatchRateAsync(String resourcegroup, String cluster, + DispatchRate dispatchRate) { + WebTarget target = + adminResourceGroups.path(resourcegroup).path("replicatorDispatchRate").queryParam("cluster", cluster); + return asyncPostRequest(target, Entity.entity(dispatchRate, MediaType.APPLICATION_JSON)); + } + + @Override + public void removeReplicatorDispatchRate(String resourcegroup, String cluster) throws PulsarAdminException { + sync(() -> removeReplicatorDispatchRateAsync(resourcegroup, cluster)); + } + + @Override + public CompletableFuture removeReplicatorDispatchRateAsync(String resourcegroup, String cluster) { + WebTarget target = + adminResourceGroups.path(resourcegroup).path("replicatorDispatchRate").queryParam("cluster", cluster); + return asyncDeleteRequest(target); + } + + @Override + public DispatchRate getReplicatorDispatchRate(String resourcegroup, String cluster) throws PulsarAdminException { + return sync(() -> getReplicatorDispatchRateAsync(resourcegroup, cluster)); + } + + @Override + public CompletableFuture getReplicatorDispatchRateAsync(String resourcegroup, String cluster) { + WebTarget target = + adminResourceGroups.path(resourcegroup).path("replicatorDispatchRate").queryParam("cluster", cluster); + return asyncGetRequest(target, DispatchRate.class); + } } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicPoliciesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicPoliciesImpl.java index 30f515228b5c7..7fffa3a2b20e0 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicPoliciesImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicPoliciesImpl.java @@ -1109,9 +1109,21 @@ public DispatchRate getReplicatorDispatchRate(String topic, boolean applied) thr @Override public CompletableFuture getReplicatorDispatchRateAsync(String topic, boolean applied) { + return getReplicatorDispatchRateAsync(topic, null, applied); + } + + @Override + public DispatchRate getReplicatorDispatchRate(String topic, String cluster, boolean applied) + throws PulsarAdminException { + return sync(() -> getReplicatorDispatchRateAsync(topic, cluster, applied)); + } + + @Override + public CompletableFuture getReplicatorDispatchRateAsync(String topic, String cluster, + boolean applied) { TopicName topicName = validateTopic(topic); - WebTarget path = topicPath(topicName, "replicatorDispatchRate"); - path = path.queryParam("applied", applied); + WebTarget path = topicPath(topicName, "replicatorDispatchRate").queryParam("applied", applied) + .queryParam("cluster", cluster); return asyncGetRequest(path, new FutureCallback(){}); } @@ -1122,8 +1134,20 @@ public void setReplicatorDispatchRate(String topic, DispatchRate dispatchRate) t @Override public CompletableFuture setReplicatorDispatchRateAsync(String topic, DispatchRate dispatchRate) { + return setReplicatorDispatchRateAsync(topic, null, dispatchRate); + } + + @Override + public void setReplicatorDispatchRate(String topic, String cluster, DispatchRate dispatchRate) + throws PulsarAdminException { + sync(() -> setReplicatorDispatchRateAsync(topic, cluster, dispatchRate)); + } + + @Override + public CompletableFuture setReplicatorDispatchRateAsync(String topic, String cluster, + DispatchRate dispatchRate) { TopicName tn = validateTopic(topic); - WebTarget path = topicPath(tn, "replicatorDispatchRate"); + WebTarget path = topicPath(tn, "replicatorDispatchRate").queryParam("cluster", cluster); return asyncPostRequest(path, Entity.entity(dispatchRate, MediaType.APPLICATION_JSON)); } @@ -1134,8 +1158,18 @@ public void removeReplicatorDispatchRate(String topic) throws PulsarAdminExcepti @Override public CompletableFuture removeReplicatorDispatchRateAsync(String topic) { + return removeReplicatorDispatchRateAsync(topic, null); + } + + @Override + public void removeReplicatorDispatchRate(String topic, String cluster) throws PulsarAdminException { + sync(() -> removeReplicatorDispatchRateAsync(topic, cluster)); + } + + @Override + public CompletableFuture removeReplicatorDispatchRateAsync(String topic, String cluster) { TopicName tn = validateTopic(topic); - WebTarget path = topicPath(tn, "replicatorDispatchRate"); + WebTarget path = topicPath(tn, "replicatorDispatchRate").queryParam("cluster", cluster); return asyncDeleteRequest(path); } diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java index 675eca867a3db..14f796940f0cf 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java @@ -72,6 +72,8 @@ import org.apache.pulsar.client.admin.ProxyStats; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminBuilder; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.admin.ResourceGroups; import org.apache.pulsar.client.admin.ResourceQuotas; import org.apache.pulsar.client.admin.Schemas; import org.apache.pulsar.client.admin.Tenants; @@ -483,17 +485,18 @@ public void namespaces() throws Exception { verify(mockNamespaces).deleteBookieAffinityGroup("myprop/clust/ns1"); namespaces.run(split("set-replicator-dispatch-rate myprop/clust/ns1 -md 10 -bd 11 -dt 12")); - verify(mockNamespaces).setReplicatorDispatchRate("myprop/clust/ns1", DispatchRate.builder() - .dispatchThrottlingRateInMsg(10) - .dispatchThrottlingRateInByte(11) - .ratePeriodInSecond(12) - .build()); + verify(mockNamespaces).setReplicatorDispatchRate("myprop/clust/ns1", null, + DispatchRate.builder() + .dispatchThrottlingRateInMsg(10) + .dispatchThrottlingRateInByte(11) + .ratePeriodInSecond(12) + .build()); namespaces.run(split("get-replicator-dispatch-rate myprop/clust/ns1")); - verify(mockNamespaces).getReplicatorDispatchRate("myprop/clust/ns1"); + verify(mockNamespaces).getReplicatorDispatchRate("myprop/clust/ns1", null); namespaces.run(split("remove-replicator-dispatch-rate myprop/clust/ns1")); - verify(mockNamespaces).removeReplicatorDispatchRate("myprop/clust/ns1"); + verify(mockNamespaces).removeReplicatorDispatchRate("myprop/clust/ns1", null); assertFalse(namespaces.run(split("unload myprop/clust/ns1 -d broker"))); @@ -1143,15 +1146,16 @@ public void topicPolicies() throws Exception { cmdTopics.run(split("set-replicator-dispatch-rate persistent://myprop/clust/ns1/ds1 -md -1 -bd -1 -dt 2")); verify(mockTopicsPolicies).setReplicatorDispatchRate("persistent://myprop/clust/ns1/ds1", + null, DispatchRate.builder() .dispatchThrottlingRateInMsg(-1) .dispatchThrottlingRateInByte(-1) .ratePeriodInSecond(2) .build()); cmdTopics.run(split("get-replicator-dispatch-rate persistent://myprop/clust/ns1/ds1")); - verify(mockTopicsPolicies).getReplicatorDispatchRate("persistent://myprop/clust/ns1/ds1", false); + verify(mockTopicsPolicies).getReplicatorDispatchRate("persistent://myprop/clust/ns1/ds1", null, false); cmdTopics.run(split("remove-replicator-dispatch-rate persistent://myprop/clust/ns1/ds1")); - verify(mockTopicsPolicies).removeReplicatorDispatchRate("persistent://myprop/clust/ns1/ds1"); + verify(mockTopicsPolicies).removeReplicatorDispatchRate("persistent://myprop/clust/ns1/ds1", null); cmdTopics.run(split("set-subscription-dispatch-rate persistent://myprop/clust/ns1/ds1 -md -1 -bd -1 -dt 2")); verify(mockTopicsPolicies).setSubscriptionDispatchRate("persistent://myprop/clust/ns1/ds1", @@ -1496,15 +1500,16 @@ public void topicPolicies() throws Exception { cmdTopics.run(split("set-replicator-dispatch-rate persistent://myprop/clust/ns1/ds1 -md -1 -bd -1 " + "-dt 2 -g")); verify(mockGlobalTopicsPolicies).setReplicatorDispatchRate("persistent://myprop/clust/ns1/ds1", + null, DispatchRate.builder() .dispatchThrottlingRateInMsg(-1) .dispatchThrottlingRateInByte(-1) .ratePeriodInSecond(2) .build()); cmdTopics.run(split("get-replicator-dispatch-rate persistent://myprop/clust/ns1/ds1 -g")); - verify(mockGlobalTopicsPolicies).getReplicatorDispatchRate("persistent://myprop/clust/ns1/ds1", false); + verify(mockGlobalTopicsPolicies).getReplicatorDispatchRate("persistent://myprop/clust/ns1/ds1", null, false); cmdTopics.run(split("remove-replicator-dispatch-rate persistent://myprop/clust/ns1/ds1 -g")); - verify(mockGlobalTopicsPolicies).removeReplicatorDispatchRate("persistent://myprop/clust/ns1/ds1"); + verify(mockGlobalTopicsPolicies).removeReplicatorDispatchRate("persistent://myprop/clust/ns1/ds1", null); cmdTopics.run(split("set-subscription-dispatch-rate persistent://myprop/clust/ns1/ds1 -md -1 -bd -1 " + "-dt 2 -g")); @@ -2768,6 +2773,78 @@ public void close() { } } + @Test + public void testReplicatorDispatchRateWithClusterByCmdNamespace() throws Exception { + PulsarAdmin admin = Mockito.mock(PulsarAdmin.class); + Namespaces mockNamespaces = mock(Namespaces.class); + when(admin.namespaces()).thenReturn(mockNamespaces); + Lookup mockLookup = mock(Lookup.class); + when(admin.lookups()).thenReturn(mockLookup); + + CmdNamespaces namespaces = new CmdNamespaces(() -> admin); + + namespaces.run(split("set-replicator-dispatch-rate tenant1/ns1 -md 10 -bd 11 -dt 12 --cluster r1")); + verify(mockNamespaces).setReplicatorDispatchRate("tenant1/ns1", "r1", + DispatchRate.builder() + .dispatchThrottlingRateInMsg(10) + .dispatchThrottlingRateInByte(11) + .ratePeriodInSecond(12) + .build()); + + namespaces.run(split("get-replicator-dispatch-rate tenant1/ns1 --cluster r1")); + verify(mockNamespaces).getReplicatorDispatchRate("tenant1/ns1", "r1"); + + namespaces.run(split("remove-replicator-dispatch-rate tenant1/ns1 --cluster r1")); + verify(mockNamespaces).removeReplicatorDispatchRate("tenant1/ns1", "r1"); + } + + @Test + public void testReplicatorDispatchRateWithClusterByCmdTopicPolicies() throws Exception { + PulsarAdmin admin = Mockito.mock(PulsarAdmin.class); + TopicPolicies mockTopicPolicies = mock(TopicPolicies.class); + when(admin.topicPolicies(anyBoolean())).thenReturn(mockTopicPolicies); + Lookup mockLookup = mock(Lookup.class); + when(admin.lookups()).thenReturn(mockLookup); + + CmdTopicPolicies topicPolicies = new CmdTopicPolicies(() -> admin); + + topicPolicies.run(split("set-replicator-dispatch-rate tenant1/ns1/topic1 -md 10 -bd 11 -dt 12 --cluster r1")); + verify(mockTopicPolicies).setReplicatorDispatchRate("persistent://tenant1/ns1/topic1", "r1", + DispatchRate.builder() + .dispatchThrottlingRateInMsg(10) + .dispatchThrottlingRateInByte(11) + .ratePeriodInSecond(12) + .build()); + + topicPolicies.run(split("get-replicator-dispatch-rate tenant1/ns1/topic1 --cluster r1")); + verify(mockTopicPolicies).getReplicatorDispatchRate("persistent://tenant1/ns1/topic1", "r1", false); + + topicPolicies.run(split("remove-replicator-dispatch-rate tenant1/ns1/topic1 --cluster r1")); + verify(mockTopicPolicies).removeReplicatorDispatchRate("persistent://tenant1/ns1/topic1", "r1"); + } + + @Test + public void testResourceGroupReplicatorDispatchRate() throws PulsarAdminException { + PulsarAdmin admin = Mockito.mock(PulsarAdmin.class); + ResourceGroups mockResourceGroups = mock(ResourceGroups.class); + when(admin.resourcegroups()).thenReturn(mockResourceGroups); + + CmdResourceGroups cmdResourceGroups = new CmdResourceGroups(() -> admin); + + cmdResourceGroups.run(split("set-replicator-dispatch-rate rg1 -md 10 -bd 11 --cluster r1")); + verify(mockResourceGroups).setReplicatorDispatchRate("rg1", "r1", + DispatchRate.builder() + .dispatchThrottlingRateInMsg(10) + .dispatchThrottlingRateInByte(11) + .build()); + + cmdResourceGroups.run(split("get-replicator-dispatch-rate rg1 --cluster r1")); + verify(mockResourceGroups).getReplicatorDispatchRate("rg1", "r1"); + + cmdResourceGroups.run(split("remove-replicator-dispatch-rate rg1 --cluster r1")); + verify(mockResourceGroups).removeReplicatorDispatchRate("rg1", "r1"); + } + public static class SchemaDemo { public SchemaDemo() { } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 54f6aa7214f0d..32d798e4dda2b 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -1193,10 +1193,14 @@ private class SetReplicatorDispatchRate extends CliCommand { + "(default 1 second will be overwrite if not passed)", required = false) private int dispatchRatePeriodSec = 1; + @Option(names = "--cluster", description = "The remote cluster for which set the dispatch rate, default to" + + " null", required = false) + private String cluster = null; + @Override void run() throws PulsarAdminException { String namespace = validateNamespace(namespaceName); - getAdmin().namespaces().setReplicatorDispatchRate(namespace, + getAdmin().namespaces().setReplicatorDispatchRate(namespace, cluster, DispatchRate.builder() .dispatchThrottlingRateInMsg(msgDispatchRate) .dispatchThrottlingRateInByte(byteDispatchRate) @@ -1211,10 +1215,14 @@ private class GetReplicatorDispatchRate extends CliCommand { @Parameters(description = "tenant/namespace", arity = "1") private String namespaceName; + @Option(names = "--cluster", description = "The remote cluster for which set the dispatch rate, default to" + + " null", required = false) + private String cluster = null; + @Override void run() throws PulsarAdminException { String namespace = validateNamespace(namespaceName); - print(getAdmin().namespaces().getReplicatorDispatchRate(namespace)); + print(getAdmin().namespaces().getReplicatorDispatchRate(namespace, cluster)); } } @@ -1224,10 +1232,14 @@ private class RemoveReplicatorDispatchRate extends CliCommand { @Parameters(description = "tenant/namespace", arity = "1") private String namespaceName; + @Option(names = "--cluster", description = "The remote cluster for which set the dispatch rate, default to" + + " null", required = false) + private String cluster = null; + @Override void run() throws PulsarAdminException { String namespace = validateNamespace(namespaceName); - getAdmin().namespaces().removeReplicatorDispatchRate(namespace); + getAdmin().namespaces().removeReplicatorDispatchRate(namespace, cluster); } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceGroups.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceGroups.java index 6ee8d7a4764a0..86c6846499873 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceGroups.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceGroups.java @@ -21,6 +21,7 @@ import java.util.function.Supplier; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.ResourceGroup; import picocli.CommandLine.Command; import picocli.CommandLine.Option; @@ -73,6 +74,14 @@ private class Create extends CliCommand { + "(default -1 will be overwrite if not passed)", required = false) private Long dispatchRateInBytes; + @Option(names = {"--replication-msg-dispatch-rate"}, description = "replication-msg-dispatch-rate " + + "(default -1 will be overwrite if not passed)", required = false) + private Long replicationDispatchRateInMsgs; + + @Option(names = {"--replication-byte-dispatch-rate"}, description = "replication-byte-dispatch-rate " + + "(default -1 will be overwrite if not passed)", required = false) + private Long replicationDispatchRateInBytes; + @Override void run() throws PulsarAdminException { ResourceGroup resourcegroup = new ResourceGroup(); @@ -80,6 +89,8 @@ void run() throws PulsarAdminException { resourcegroup.setDispatchRateInBytes(dispatchRateInBytes); resourcegroup.setPublishRateInMsgs(publishRateInMsgs); resourcegroup.setPublishRateInBytes(publishRateInBytes); + resourcegroup.setReplicationDispatchRateInMsgs(replicationDispatchRateInMsgs); + resourcegroup.setReplicationDispatchRateInBytes(replicationDispatchRateInBytes); getAdmin().resourcegroups().createResourceGroup(resourceGroupName, resourcegroup); } } @@ -106,6 +117,14 @@ private class Update extends CliCommand { "-bd" }, description = "byte-dispatch-rate ", required = false) private Long dispatchRateInBytes; + @Option(names = {"--replication-msg-dispatch-rate"}, description = "replication-msg-dispatch-rate " + + "(default -1 will be overwrite if not passed)", required = false) + private Long replicationDispatchRateInMsgs; + + @Option(names = {"--replication-byte-dispatch-rate"}, description = "replication-byte-dispatch-rate " + + "(default -1 will be overwrite if not passed)", required = false) + private Long replicationDispatchRateInBytes; + @Override void run() throws PulsarAdminException { ResourceGroup resourcegroup = new ResourceGroup(); @@ -113,6 +132,8 @@ void run() throws PulsarAdminException { resourcegroup.setDispatchRateInBytes(dispatchRateInBytes); resourcegroup.setPublishRateInMsgs(publishRateInMsgs); resourcegroup.setPublishRateInBytes(publishRateInBytes); + resourcegroup.setReplicationDispatchRateInMsgs(replicationDispatchRateInMsgs); + resourcegroup.setReplicationDispatchRateInBytes(replicationDispatchRateInBytes); getAdmin().resourcegroups().updateResourceGroup(resourceGroupName, resourcegroup); } @@ -129,6 +150,65 @@ void run() throws PulsarAdminException { } } + @Command(description = "Set replicator rate limiter for a resourcegroup") + private class SetReplicatorDispatchRate extends CliCommand { + @Parameters(description = "resourcegroup-name", arity = "1") + private String resourceGroupName; + + @Option(names = {"--msg-dispatch-rate", + "-md"}, description = "message-dispatch-rate " + + "(default -1 will be overwrite if not passed)", required = false) + private int msgDispatchRate = -1; + + @Option(names = {"--byte-dispatch-rate", + "-bd"}, description = "byte-dispatch-rate (default -1 will be overwrite if not passed)", + required = false) + private long byteDispatchRate = -1; + + @Option(names = "--cluster", description = "The remote cluster for which set the dispatch rate", + required = true) + private String cluster; + + @Override + void run() throws PulsarAdminException { + getAdmin().resourcegroups().setReplicatorDispatchRate(resourceGroupName, cluster, + DispatchRate.builder() + .dispatchThrottlingRateInMsg(msgDispatchRate) + .dispatchThrottlingRateInByte(byteDispatchRate) + .build()); + } + } + + @Command(description ="Get replicator rate limiter from a resourcegroup") + private class GetReplicatorDispatchRate extends CliCommand { + @Parameters(description = "resourcegroup-name", arity = "1") + private String resourceGroupName; + + @Option(names = "--cluster", description = "The remote cluster for which set the dispatch rate", + required = true) + private String cluster; + + @Override + void run() throws PulsarAdminException { + print(getAdmin().resourcegroups().getReplicatorDispatchRate(resourceGroupName, cluster)); + } + } + + @Command(description = "Remove replicator rate limiter from a resourcegroup") + private class RemoveReplicatorDispatchRate extends CliCommand { + @Parameters(description = "resourcegroup-name", arity = "1") + private String resourceGroupName; + + @Option(names = "--cluster", description = "The remote cluster for which set the dispatch rate", + required = true) + private String cluster; + + @Override + void run() throws PulsarAdminException { + getAdmin().resourcegroups().removeReplicatorDispatchRate(resourceGroupName, cluster); + } + } + public CmdResourceGroups(Supplier admin) { super("resourcegroups", admin); @@ -137,6 +217,9 @@ public CmdResourceGroups(Supplier admin) { addCommand("create", new CmdResourceGroups.Create()); addCommand("update", new CmdResourceGroups.Update()); addCommand("delete", new CmdResourceGroups.Delete()); + addCommand("get-replicator-dispatch-rate", new GetReplicatorDispatchRate()); + addCommand("set-replicator-dispatch-rate", new SetReplicatorDispatchRate()); + addCommand("remove-replicator-dispatch-rate", new RemoveReplicatorDispatchRate()); } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopicPolicies.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopicPolicies.java index 6d72b929cbf39..b79ffe6ac3960 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopicPolicies.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopicPolicies.java @@ -1468,10 +1468,14 @@ private class GetReplicatorDispatchRate extends CliCommand { + "If set to true, broker returned global topic policies") private boolean isGlobal = false; + @Option(names = "--cluster", description = "The remote cluster for which set the dispatch rate, default to" + + " null", required = false) + private String cluster = null; + @Override void run() throws PulsarAdminException { String persistentTopic = validatePersistentTopic(topicName); - print(getTopicPolicies(isGlobal).getReplicatorDispatchRate(persistentTopic, applied)); + print(getTopicPolicies(isGlobal).getReplicatorDispatchRate(persistentTopic, cluster, applied)); } } @@ -1502,10 +1506,14 @@ private class SetReplicatorDispatchRate extends CliCommand { + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal = false; + @Option(names = "--cluster", description = "The remote cluster for which set the dispatch rate, default to" + + " null", required = false) + private String cluster = null; + @Override void run() throws PulsarAdminException { String persistentTopic = validatePersistentTopic(topicName); - getTopicPolicies(isGlobal).setReplicatorDispatchRate(persistentTopic, + getTopicPolicies(isGlobal).setReplicatorDispatchRate(persistentTopic, cluster, DispatchRate.builder() .dispatchThrottlingRateInMsg(msgDispatchRate) .dispatchThrottlingRateInByte(byteDispatchRate) @@ -1524,10 +1532,14 @@ private class RemoveReplicatorDispatchRate extends CliCommand { + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal = false; + @Option(names = "--cluster", description = "The remote cluster for which set the dispatch rate, default to" + + " null", required = false) + private String cluster = null; + @Override void run() throws PulsarAdminException { String persistentTopic = validatePersistentTopic(topicName); - getTopicPolicies(isGlobal).removeReplicatorDispatchRate(persistentTopic); + getTopicPolicies(isGlobal).removeReplicatorDispatchRate(persistentTopic, cluster); } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java index 920eddcffbf72..20e94b16effe5 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java @@ -22,6 +22,7 @@ import java.util.EnumSet; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import lombok.Getter; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType; @@ -52,7 +53,7 @@ public class HierarchyTopicPolicies { final PolicyHierarchyValue dispatcherPauseOnAckStatePersistentEnabled; final PolicyHierarchyValue delayedDeliveryTickTimeMillis; final PolicyHierarchyValue delayedDeliveryMaxDelayInMillis; - final PolicyHierarchyValue replicatorDispatchRate; + final Map> replicatorDispatchRate; final PolicyHierarchyValue maxConsumersPerSubscription; final PolicyHierarchyValue subscribeRate; final PolicyHierarchyValue subscriptionDispatchRate; @@ -63,6 +64,7 @@ public class HierarchyTopicPolicies { final PolicyHierarchyValue entryFilters; final PolicyHierarchyValue replicateSubscriptionState; + final PolicyHierarchyValue resourceGroupName; public HierarchyTopicPolicies() { replicationClusters = new PolicyHierarchyValue<>(); @@ -88,7 +90,7 @@ public HierarchyTopicPolicies() { dispatcherPauseOnAckStatePersistentEnabled = new PolicyHierarchyValue<>(); delayedDeliveryTickTimeMillis = new PolicyHierarchyValue<>(); delayedDeliveryMaxDelayInMillis = new PolicyHierarchyValue<>(); - replicatorDispatchRate = new PolicyHierarchyValue<>(); + replicatorDispatchRate = new ConcurrentHashMap<>(); compactionThreshold = new PolicyHierarchyValue<>(); subscribeRate = new PolicyHierarchyValue<>(); subscriptionDispatchRate = new PolicyHierarchyValue<>(); @@ -97,5 +99,6 @@ public HierarchyTopicPolicies() { schemaValidationEnforced = new PolicyHierarchyValue<>(); entryFilters = new PolicyHierarchyValue<>(); replicateSubscriptionState = new PolicyHierarchyValue<>(); + resourceGroupName = new PolicyHierarchyValue<>(); } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicOperation.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicOperation.java index 4c74ffcd8f085..9511847cc8fb9 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicOperation.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicOperation.java @@ -56,4 +56,7 @@ public enum TopicOperation { SET_REPLICATED_SUBSCRIPTION_STATUS, GET_REPLICATED_SUBSCRIPTION_STATUS, TRIM_TOPIC, + + SET_RESOURCE_GROUP, + GET_RESOURCE_GROUP, } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java index 46543b48c0ec5..df6cf0222e76c 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java @@ -75,7 +75,13 @@ public class TopicPolicies implements Cloneable { private Integer deduplicationSnapshotIntervalSeconds; private Integer maxMessageSize; private Integer maxSubscriptionsPerTopic; + /** + * @deprecated Use {@link #replicatorDispatchRateMap} instead. + */ + @Deprecated private DispatchRateImpl replicatorDispatchRate; + @Builder.Default + private Map replicatorDispatchRateMap = new HashMap<>(); private SchemaCompatibilityStrategy schemaCompatibilityStrategy; private EntryFilters entryFilters; // If set, it will override the namespace settings for allowing auto subscription creation @@ -91,6 +97,8 @@ public class TopicPolicies implements Cloneable { private Boolean replicateSubscriptionState; + private String resourceGroupName; + @SneakyThrows @Override public TopicPolicies clone() { @@ -149,6 +157,11 @@ public TopicPolicies clone() { cloned.subscriptionPolicies = new HashMap<>(); } + if (this.replicatorDispatchRateMap != null) { + cloned.replicatorDispatchRateMap = new HashMap<>(); + cloned.replicatorDispatchRateMap.putAll(this.replicatorDispatchRateMap); + } + // Primitive types (Boolean, Integer, Long, String) and enums (SchemaCompatibilityStrategy) // are fine with the shallow copy from super.clone(). // isGlobal, deduplicationEnabled, messageTTLInSeconds, maxProducerPerTopic, etc. diff --git a/pulsar-common/src/main/resources/findbugsExclude.xml b/pulsar-common/src/main/resources/findbugsExclude.xml index b3e511006bce3..6867fd72cbbea 100644 --- a/pulsar-common/src/main/resources/findbugsExclude.xml +++ b/pulsar-common/src/main/resources/findbugsExclude.xml @@ -58,4 +58,8 @@ + + + + From ac596fe75222021745f92bf9b8b6306be1812079 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Thu, 26 Mar 2026 15:35:34 +0800 Subject: [PATCH 07/16] [feat][broker] Introduce structured event data models for topic and message lifecycle events --- .../bookkeeper/mledger/ManagedLedger.java | 4 + .../mledger/ManagedLedgerEventListener.java | 46 +++ .../mledger/impl/ManagedLedgerImpl.java | 62 ++++- .../bookkeeper/mledger/impl/OpAddEntry.java | 5 +- .../mledger/impl/ManagedCursorTest.java | 5 +- .../mledger/impl/ManagedLedgerTest.java | 85 +++++- .../pulsar/broker/admin/AdminResource.java | 45 ++- .../admin/impl/PersistentTopicsBase.java | 179 +++++++++--- .../event/data/ConsumerConnectEventData.java | 42 +++ .../data/ConsumerDisconnectEventData.java | 39 +++ .../event/data/DisconnectInitiator.java | 24 ++ .../event/data/LedgerPurgeEventData.java | 44 +++ .../event/data/LedgerRollEventData.java | 35 +++ .../event/data/MessageExpireEventData.java | 34 +++ .../event/data/ProducerConnectEventData.java | 38 +++ .../data/ProducerDisconnectEventData.java | 37 +++ .../event/data/ReplicatorStartEventData.java | 35 +++ .../event/data/ReplicatorStopEventData.java | 35 +++ .../SubscriptionClearBacklogEventData.java | 37 +++ .../data/SubscriptionCreateEventData.java | 38 +++ .../data/SubscriptionDeleteEventData.java | 41 +++ .../event/data/SubscriptionSeekEventData.java | 39 +++ .../event/data/TopicCreateEventData.java | 35 +++ .../event/data/TopicDeleteEventData.java | 33 +++ .../event/data/TopicLookupEventData.java | 40 +++ .../data/TopicMetadataUpdateEventData.java | 34 +++ .../data/TopicPoliciesApplyEventData.java | 35 +++ .../data/TopicPoliciesUpdateEventData.java | 34 +++ .../broker/event/data/package-info.java | 20 ++ .../pulsar/broker/lookup/LookupResult.java | 4 + .../pulsar/broker/lookup/TopicLookupBase.java | 99 ++++++- .../resourcegroup/ResourceGroupService.java | 4 +- .../broker/service/AbstractReplicator.java | 12 + .../pulsar/broker/service/BrokerService.java | 263 +++++++++++++----- .../pulsar/broker/service/Consumer.java | 4 +- .../pulsar/broker/service/ServerCnx.java | 179 +++++++++++- .../pulsar/broker/service/Subscription.java | 24 ++ .../SystemTopicBasedTopicPoliciesService.java | 7 + .../broker/service/TopicEventsDispatcher.java | 152 +++++++++- .../broker/service/TopicEventsListener.java | 74 ++++- .../broker/service/TopicLoadingContext.java | 20 +- .../PersistentMessageExpiryMonitor.java | 24 +- .../persistent/PersistentReplicator.java | 27 +- .../persistent/PersistentSubscription.java | 43 ++- .../service/persistent/PersistentTopic.java | 48 ++++ .../pulsar/broker/web/PulsarWebResource.java | 9 +- .../MangedLedgerInterceptorImpl2Test.java | 5 +- .../broker/TopicEventsListenerTest.java | 223 +++++++++++---- .../AdminReplicatorDispatchRateTest.java | 3 +- .../apache/pulsar/broker/admin/AdminTest.java | 3 +- .../broker/admin/ResourceGroupsTest.java | 6 +- .../RGUsageMTAggrWaitForAllMsgsTest.java | 12 +- .../ResourceGroupMetricTest.java | 12 +- .../ResourceGroupRateLimiterManagerTest.java | 12 +- .../ResourceGroupServiceTest.java | 61 ++-- ...GroupUsageAggregationOnTopicLevelTest.java | 33 +-- .../broker/service/BrokerServiceTest.java | 8 +- .../service/PersistentMessageFinderTest.java | 46 +-- .../broker/service/PersistentTopicTest.java | 13 +- .../service/ReplicatorRateLimiterTest.java | 111 ++++---- .../pulsar/broker/service/ServerCnxTest.java | 2 + .../buffer/TopicTransactionBufferTest.java | 5 +- .../api/NonDurableSubscriptionTest.java | 17 +- .../pulsar/admin/cli/CmdResourceGroups.java | 2 +- .../policies/data/HierarchyTopicPolicies.java | 2 + .../policies/data/PolicyHierarchyValue.java | 2 + 66 files changed, 2310 insertions(+), 441 deletions(-) create mode 100644 managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerEventListener.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ConsumerConnectEventData.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ConsumerDisconnectEventData.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/DisconnectInitiator.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/LedgerPurgeEventData.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/LedgerRollEventData.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/MessageExpireEventData.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ProducerConnectEventData.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ProducerDisconnectEventData.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ReplicatorStartEventData.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ReplicatorStopEventData.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/SubscriptionClearBacklogEventData.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/SubscriptionCreateEventData.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/SubscriptionDeleteEventData.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/SubscriptionSeekEventData.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicCreateEventData.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicDeleteEventData.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicLookupEventData.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicMetadataUpdateEventData.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicPoliciesApplyEventData.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicPoliciesUpdateEventData.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/package-info.java diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java index 16b31b979cfaf..67f5f4af65705 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java @@ -770,4 +770,8 @@ default long getLastAddEntryTime() { default long getMetadataCreationTimestamp() { return 0; } + + default void addLedgerEventListener(ManagedLedgerEventListener listener) { + // No-op by default + } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerEventListener.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerEventListener.java new file mode 100644 index 0000000000000..e650d71138ec9 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerEventListener.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.Value; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo; + +public interface ManagedLedgerEventListener { + enum LedgerRollReason { + FULL, // Ledger is full based on size or time + INACTIVE, // No writes for a while + APPEND_FAIL, ConcurrentModification, // Failed to append to the ledger + } + + @AllArgsConstructor + @NoArgsConstructor(force = true) + @Builder + @Value + class LedgerRollEvent { + long ledgerId; + LedgerRollReason reason; + } + + void onLedgerRoll(LedgerRollEvent event); + + void onLedgerDelete(LedgerInfo... ledgerInfos); +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 95ba53312b45d..d1d25acfe42a6 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -54,6 +54,7 @@ import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; @@ -102,6 +103,9 @@ import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerAttributes; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.mledger.ManagedLedgerEventListener; +import org.apache.bookkeeper.mledger.ManagedLedgerEventListener.LedgerRollEvent; +import org.apache.bookkeeper.mledger.ManagedLedgerEventListener.LedgerRollReason; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerException.BadVersionException; import org.apache.bookkeeper.mledger.ManagedLedgerException.CursorNotFoundException; @@ -1800,7 +1804,7 @@ synchronized void addEntryFailedDueToConcurrentlyModified(final LedgerHandle cur + " The last add confirmed position in memory is {}, and the value" + " stored in metadata store is {}.", name, lh.getId(), currentLedger.getLastAddConfirmed(), lh.getLastAddConfirmed()); - ledgerClosed(currentLedger, lh.getLastAddConfirmed()); + ledgerClosed(currentLedger, lh.getLastAddConfirmed(), LedgerRollReason.ConcurrentModification); } else { log.error("[{}] Fencing the topic to ensure durability and consistency(the current ledger was" + " concurrent modified by a other bookie client, which is not expected)." @@ -1815,14 +1819,15 @@ synchronized void addEntryFailedDueToConcurrentlyModified(final LedgerHandle cur }, null, true); } - synchronized void ledgerClosed(final LedgerHandle lh) { - ledgerClosed(lh, null); + @VisibleForTesting + public synchronized void ledgerClosedWithReason(final LedgerHandle lh, LedgerRollReason ledgerRollReason) { + ledgerClosed(lh, null, ledgerRollReason); } // ////////////////////////////////////////////////////////////////////// // Private helpers - synchronized void ledgerClosed(final LedgerHandle lh, Long lastAddConfirmed) { + synchronized void ledgerClosed(final LedgerHandle lh, Long lastAddConfirmed, LedgerRollReason ledgerRollReason) { final State state = STATE_UPDATER.get(this); LedgerHandle currentLedger = this.currentLedger; if (currentLedger == lh && (state == State.ClosingLedger || state == State.LedgerOpened)) { @@ -1855,7 +1860,7 @@ synchronized void ledgerClosed(final LedgerHandle lh, Long lastAddConfirmed) { maybeOffloadInBackground(NULL_OFFLOAD_PROMISE); - createLedgerAfterClosed(); + createLedgerAfterClosed(ledgerRollReason); } @Override @@ -1865,12 +1870,17 @@ public void skipNonRecoverableLedger(long ledgerId){ } } - synchronized void createLedgerAfterClosed() { + @VisibleForTesting + public synchronized void createLedgerAfterClosed(LedgerRollReason ledgerRollReason) { if (isNeededCreateNewLedgerAfterCloseLedger()) { log.info("[{}] Creating a new ledger after closed {}", name, currentLedger == null ? "null" : currentLedger.getId()); STATE_UPDATER.set(this, State.CreatingLedger); this.lastLedgerCreationInitiationTimestamp = System.currentTimeMillis(); + notifyRollLedgerEvent(LedgerRollEvent.builder() + .ledgerId(currentLedger == null ? -1 : currentLedger.getId()) + .reason(ledgerRollReason) + .build()); mbean.startDataLedgerCreateOp(); // Use the executor here is to avoid use the Zookeeper thread to create the ledger which will lead // to deadlock at the zookeeper client, details to see https://github.com/apache/pulsar/issues/13736 @@ -1909,7 +1919,7 @@ public void closeComplete(int rc, LedgerHandle lh, Object o) { name, lh.getId(), BKException.getMessage(rc)); } - ledgerClosed(lh); + ledgerClosedWithReason(lh, LedgerRollReason.FULL); } }, null); } @@ -3045,10 +3055,13 @@ public void operationComplete(Void result, Stat stat) { metadataMutex.unlock(); trimmerMutex.unlock(); + notifyDeleteLedgerEvent(ledgersToDelete.toArray(new LedgerInfo[0])); for (LedgerInfo ls : ledgersToDelete) { log.info("[{}] Removing ledger {} - size: {}", name, ls.getLedgerId(), ls.getSize()); asyncDeleteLedger(ls.getLedgerId(), ls); } + + notifyDeleteLedgerEvent(offloadedLedgersToDelete.toArray(new LedgerInfo[0])); for (LedgerInfo ls : offloadedLedgersToDelete) { log.info("[{}] Deleting offloaded ledger {} from bookkeeper - size: {}", name, ls.getLedgerId(), ls.getSize()); @@ -4719,14 +4732,15 @@ public void clearBacklogFailed(ManagedLedgerException exception, Object ctx) { }, null); futures.add(future); } - CompletableFuture future = new CompletableFuture(); + CompletableFuture> future = new CompletableFuture(); FutureUtil.waitForAll(futures).thenAccept(p -> { internalTrimLedgers(true, future); }).exceptionally(e -> { future.completeExceptionally(e); return null; }); - return future; + return future.thenRun(() -> { + }); } @Override @@ -4907,7 +4921,7 @@ public boolean checkInactiveLedgerAndRollOver() { name, lh.getId(), BKException.getMessage(rc)); } - ledgerClosed(lh); + ledgerClosedWithReason(lh, LedgerRollReason.INACTIVE); // we do not create ledger here, since topic is inactive for a long time. }, null); return true; @@ -4983,4 +4997,32 @@ public long getLastAddEntryTime() { public long getMetadataCreationTimestamp() { return ledgersStat != null ? ledgersStat.getCreationTimestamp() : 0; } + + private final List ledgerEventListeners = new CopyOnWriteArrayList<>(); + + @Override + public void addLedgerEventListener(ManagedLedgerEventListener listener) { + Objects.requireNonNull(listener); + ledgerEventListeners.add(listener); + } + + private void notifyRollLedgerEvent(LedgerRollEvent event) { + for (ManagedLedgerEventListener listener : ledgerEventListeners) { + try { + listener.onLedgerRoll(event); + } catch (Exception e) { + log.warn("Exception in ledger rolled listener for ledger {}", event, e); + } + } + } + + private void notifyDeleteLedgerEvent(LedgerInfo... ledgerInfos) { + for (ManagedLedgerEventListener listener : ledgerEventListeners) { + try { + listener.onLedgerDelete(ledgerInfos); + } catch (Exception e) { + log.warn("Exception in ledger delete listener", e); + } + } + } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java index a1422b2f42192..e1779ae3d4db8 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java @@ -34,6 +34,7 @@ import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback; +import org.apache.bookkeeper.mledger.ManagedLedgerEventListener.LedgerRollReason; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; @@ -304,7 +305,7 @@ public void closeComplete(int rc, LedgerHandle lh, Object ctx) { log.warn("Error when closing ledger {}. Status={}", lh.getId(), BKException.getMessage(rc)); } - ml.ledgerClosed(lh); + ml.ledgerClosedWithReason(lh, LedgerRollReason.FULL); updateLatency(); AddEntryCallback cb = callbackUpdater.getAndSet(this, null); @@ -360,7 +361,7 @@ void handleAddFailure(final LedgerHandle lh, Integer rc) { || rc.intValue() == BKException.Code.LedgerFencedException)) { finalMl.addEntryFailedDueToConcurrentlyModified(lh, rc); } else { - finalMl.ledgerClosed(lh); + finalMl.ledgerClosedWithReason(lh, LedgerRollReason.APPEND_FAIL); } }); } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index 6b37c4a5c18aa..72bd81c08b025 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -104,6 +104,7 @@ import org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.mledger.ManagedLedgerEventListener.LedgerRollReason; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException; import org.apache.bookkeeper.mledger.ManagedLedgerFactory; @@ -5636,13 +5637,13 @@ public void testEstimateEntryCountBySize() throws Exception { } long ledger1 = ml.getCurrentLedger().getId(); ml.getCurrentLedger().close(); - ml.ledgerClosed(ml.getCurrentLedger()); + ml.ledgerClosedWithReason(ml.getCurrentLedger(), LedgerRollReason.FULL); for (int i = 0; i < 100; i++) { ml.addEntry(new byte[]{1, 2}); } long ledger2 = ml.getCurrentLedger().getId(); ml.getCurrentLedger().close(); - ml.ledgerClosed(ml.getCurrentLedger()); + ml.ledgerClosedWithReason(ml.getCurrentLedger(), LedgerRollReason.FULL); for (int i = 0; i < 100; i++) { ml.addEntry(new byte[]{1, 2, 3, 4}); } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index c5f75150495b1..6b679f1871e95 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -72,6 +72,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; @@ -120,6 +121,8 @@ import org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.mledger.ManagedLedgerEventListener; +import org.apache.bookkeeper.mledger.ManagedLedgerEventListener.LedgerRollReason; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException; import org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerNotFoundException; @@ -4264,7 +4267,7 @@ public void testIsNoMessagesAfterPos() throws Exception { assertTrue(ml.isNoMessagesAfterPos(PositionFactory.create(p3.getLedgerId() + 1, -1))); // More than one ledger. - ml.ledgerClosed(ml.currentLedger); + ml.ledgerClosedWithReason(ml.currentLedger, LedgerRollReason.FULL); Position p4 = ml.addEntry(data); Position p5 = ml.addEntry(data); Position p6 = ml.addEntry(data); @@ -4278,8 +4281,8 @@ public void testIsNoMessagesAfterPos() throws Exception { assertTrue(ml.isNoMessagesAfterPos(PositionFactory.create(p6.getLedgerId() + 1, -1))); // Switch ledger and make the entry id of Last confirmed entry is -1; - ml.ledgerClosed(ml.currentLedger); - ml.createLedgerAfterClosed(); + ml.ledgerClosedWithReason(ml.currentLedger, LedgerRollReason.FULL); + ml.createLedgerAfterClosed(LedgerRollReason.FULL); Awaitility.await().untilAsserted(() -> { assertEquals(ml.currentLedgerEntries, 0); }); @@ -5194,4 +5197,80 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { // Verify properties are preserved after cursor reset assertEquals(cursor.getProperties(), expectedProperties); } + + @Test + public void testListenLedgerDelete() throws Exception { + String name = "testListenLedgerDelete-" + System.nanoTime(); + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setMaxEntriesPerLedger(1); + ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open(name, config); + + List addedPositions = new CopyOnWriteArrayList<>(); + ml.addLedgerEventListener(new ManagedLedgerEventListener() { + @Override + public void onLedgerRoll(LedgerRollEvent event) { + // no-op + } + + @Override + public void onLedgerDelete(LedgerInfo... ledgerInfos) { + addedPositions.forEach(n -> { + for (LedgerInfo ledgerInfo : ledgerInfos) { + if (n.getLedgerId() == ledgerInfo.getLedgerId()) { + addedPositions.remove(n); + log.info("Removed position: {}", n); + } + } + }); + } + }); + + int messageCount = 4; + for (int i = 0; i < messageCount; i++) { + addedPositions.add(ml.addEntry(("entry-" + i).getBytes())); + } + + Awaitility.await().untilAsserted(() -> { + assertThat(addedPositions.size()).isZero(); + }); + } + + @Test + public void testListenLedgerRoll() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + int maxEntriesPerLedger = 4; + config.setMaxEntriesPerLedger(maxEntriesPerLedger); + int maximumRolloverTimeMS = 500; + config.setMaximumRolloverTime(maximumRolloverTimeMS, TimeUnit.MILLISECONDS); + config.setMinimumRolloverTime(0, TimeUnit.SECONDS); + int inactiveLedgerRollOverTimeMS = 3000; + config.setInactiveLedgerRollOverTime(inactiveLedgerRollOverTimeMS, TimeUnit.MILLISECONDS); + String name = "testListenLedgerRoll-" + System.nanoTime(); + ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open(name, config); + + AtomicBoolean rolled = new AtomicBoolean(false); + ml.addLedgerEventListener(new ManagedLedgerEventListener() { + @Override + public void onLedgerRoll(LedgerRollEvent event) { + rolled.set(true); + } + + @Override + public void onLedgerDelete(LedgerInfo... ledgerInfos) { + // no-op + } + }); + + for (int i = 0; i < maxEntriesPerLedger + 1; i++) { + ml.addEntry(("entry-" + i).getBytes()); + } + + Thread.sleep(inactiveLedgerRollOverTimeMS); // wait for the inactive + + ml.checkInactiveLedgerAndRollOver(); + + Awaitility.await().untilAsserted(() -> { + assertThat(rolled.get()).isTrue(); + }); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java index 6bdc33d9c5a15..69de0edaf4a56 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java @@ -44,8 +44,10 @@ import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authorization.AuthorizationService; +import org.apache.pulsar.broker.event.data.TopicCreateEventData; import org.apache.pulsar.broker.namespace.TopicExistsInfo; import org.apache.pulsar.broker.resources.ClusterResources; +import org.apache.pulsar.broker.service.TopicEventsDispatcher; import org.apache.pulsar.broker.service.TopicEventsListener.EventStage; import org.apache.pulsar.broker.service.TopicEventsListener.TopicEvent; import org.apache.pulsar.broker.service.TopicPoliciesService; @@ -171,25 +173,32 @@ public CompletableFuture validatePoliciesReadOnlyAccessAsync() { protected CompletableFuture tryCreatePartitionsAsync(int numPartitions) { if (!topicName.isPersistent()) { for (int i = 0; i < numPartitions; i++) { - pulsar().getBrokerService().getTopicEventsDispatcher() - .notify(topicName.getPartition(i).toString(), TopicEvent.CREATE, EventStage.SUCCESS); + newTopicEvent(topicName.getPartition(i), TopicEvent.CREATE) + .data(TopicCreateEventData.builder().partitions(numPartitions).build()) + .dispatch(); } return CompletableFuture.completedFuture(null); } List> futures = new ArrayList<>(numPartitions); for (int i = 0; i < numPartitions; i++) { - futures.add(tryCreatePartitionAsync(i)); + futures.add(tryCreatePartitionAsync(numPartitions, i)); } return FutureUtil.waitForAll(futures); } - private CompletableFuture tryCreatePartitionAsync(final int partition) { + private CompletableFuture tryCreatePartitionAsync(int numPartitions, final int partition) { CompletableFuture result = new CompletableFuture<>(); getPulsarResources().getTopicResources().createPersistentTopicAsync(topicName.getPartition(partition)) .thenAccept(r -> { if (log.isDebugEnabled()) { log.debug("[{}] Topic partition {} created.", clientAppId(), topicName.getPartition(partition)); } + newTopicEvent(topicName.getPartition(partition), TopicEvent.CREATE) + .data(TopicCreateEventData + .builder() + .partitions(numPartitions) + .build()) + .dispatch(); result.complete(null); }).exceptionally(ex -> { if (ex.getCause() instanceof AlreadyExistsException) { @@ -203,14 +212,20 @@ private CompletableFuture tryCreatePartitionAsync(final int partition) { // resource result.complete(null); } else { + newTopicEvent(topicName.getPartition(partition), TopicEvent.CREATE) + .error(ex) + .data(TopicCreateEventData + .builder() + .partitions(numPartitions) + .build()) + .stage(EventStage.FAILURE) + .dispatch(); log.error("[{}] Fail to create topic partition {}", clientAppId(), topicName.getPartition(partition), ex.getCause()); result.completeExceptionally(ex.getCause()); } return null; }); - pulsar().getBrokerService().getTopicEventsDispatcher() - .notifyOnCompletion(result, topicName.getPartition(partition).toString(), TopicEvent.CREATE); return result; } @@ -644,9 +659,14 @@ protected void internalCreatePartitionedTopic(AsyncResponse asyncResponse, int n })) .thenRun(() -> { for (int i = 0; i < numPartitions; i++) { - pulsar().getBrokerService().getTopicEventsDispatcher() - .notify(topicName.getPartition(i).toString(), TopicEvent.CREATE, - EventStage.BEFORE); + newTopicEvent(topicName.getPartition(i), TopicEvent.CREATE) + .stage(EventStage.BEFORE) + .data(TopicCreateEventData.builder() + .partitions(numPartitions) + .properties(properties) + .build() + ) + .dispatch(); } }) .thenCompose(__ -> provisionPartitionedTopicPath(numPartitions, createLocalTopicOnly, properties)) @@ -1057,4 +1077,11 @@ protected String getReplicatorDispatchRateKey(String remoteCluster) { return DispatchRateLimiter.getReplicatorDispatchRateKey(pulsar().getConfiguration().getClusterName(), remoteCluster); } + + protected TopicEventsDispatcher.TopicEventBuilder newTopicEvent(TopicName topic, TopicEvent topicEvent) { + return pulsar().getBrokerService().getTopicEventsDispatcher() + .newEvent(topic.toString(), topicEvent) + .role(clientAppId(), originalPrincipal()) + .clientVersion(getClientVersion()); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 5d4c5cb815e9f..f68ea639008a5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -57,6 +57,7 @@ import javax.ws.rs.core.StreamingOutput; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.AsyncCallbacks.ManagedLedgerInfoCallback; +import org.apache.bookkeeper.mledger.AsyncCallbacks.UpdatePropertiesCallback; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerException; @@ -75,6 +76,12 @@ import org.apache.pulsar.broker.admin.AdminResource; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.authorization.AuthorizationService; +import org.apache.pulsar.broker.event.data.SubscriptionClearBacklogEventData; +import org.apache.pulsar.broker.event.data.SubscriptionCreateEventData; +import org.apache.pulsar.broker.event.data.SubscriptionDeleteEventData; +import org.apache.pulsar.broker.event.data.SubscriptionSeekEventData; +import org.apache.pulsar.broker.event.data.TopicDeleteEventData; +import org.apache.pulsar.broker.event.data.TopicMetadataUpdateEventData; import org.apache.pulsar.broker.service.AnalyzeBacklogResult; import org.apache.pulsar.broker.service.BrokerServiceException.AlreadyRunningException; import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException; @@ -83,6 +90,9 @@ import org.apache.pulsar.broker.service.MessageExpirer; import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.TopicEventsListener.EventStage; +import org.apache.pulsar.broker.service.TopicEventsListener.TopicEvent; +import org.apache.pulsar.broker.service.TopicLoadingContext; import org.apache.pulsar.broker.service.TopicPoliciesService; import org.apache.pulsar.broker.service.persistent.PersistentReplicator; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; @@ -92,6 +102,7 @@ import org.apache.pulsar.client.admin.OffloadProcessStatus; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.admin.PulsarAdminException.ConflictException; import org.apache.pulsar.client.admin.PulsarAdminException.NotFoundException; import org.apache.pulsar.client.admin.PulsarAdminException.PreconditionFailedException; import org.apache.pulsar.client.api.MessageId; @@ -111,13 +122,13 @@ import org.apache.pulsar.common.compression.CompressionCodecProvider; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.PartitionedManagedLedgerInfo; -import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.AuthAction; import org.apache.pulsar.common.policies.data.AutoSubscriptionCreationOverride; import org.apache.pulsar.common.policies.data.BacklogQuota; +import org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType; import org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies; import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.EntryFilters; @@ -149,6 +160,7 @@ import org.apache.pulsar.common.util.collections.BitSetRecyclable; import org.apache.pulsar.compaction.Compactor; import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.metadata.api.MetadataStoreException.BadVersionException; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; import org.jspecify.annotations.NonNull; import org.slf4j.Logger; @@ -246,7 +258,7 @@ private CompletableFuture grantPermissionsAsync(TopicName topicUri, String log.warn("[{}] Failed to set permissions for topic {}: Namespace does not exist", clientAppId(), topicUri, realCause); throw new RestException(Status.NOT_FOUND, "Topic's namespace does not exist"); - } else if (realCause instanceof MetadataStoreException.BadVersionException + } else if (realCause instanceof BadVersionException || realCause instanceof IllegalStateException) { log.warn("[{}] Failed to set permissions for topic {}: {}", clientAppId(), topicUri, realCause.getMessage(), realCause); @@ -344,7 +356,14 @@ protected CompletableFuture internalCreateNonPartitionedTopicAsync(boolean log.warn("[{}] Topic {} already exists", clientAppId(), topicName); throw new RestException(Status.CONFLICT, "This topic already exists"); } - return pulsar().getBrokerService().getTopic(topicName.toString(), true, properties); + return pulsar().getBrokerService() + .getTopic(TopicLoadingContext.builder() + .topicName(topicName) + .createIfMissing(true) + .properties(properties) + .clientVersion(getClientVersion()) + .build() + ); }) .thenAccept(__ -> log.info("[{}] Successfully created non-partitioned topic {}", clientAppId(), topicName)); } @@ -426,8 +445,14 @@ protected CompletableFuture internalCreateNonPartitionedTopicAsync(boolean // update current cluster topic metadata : namespaceResources().getPartitionedTopicResources() .updatePartitionedTopicAsync(topicName, m -> - new PartitionedTopicMetadata(expectPartitions, m.properties)); - return updateMetadataFuture + new PartitionedTopicMetadata(expectPartitions, m.properties)) + .thenRun(() -> newTopicEvent(topicName, TopicEvent.TOPIC_METADATA_UPDATE) + .data(TopicMetadataUpdateEventData.builder() + .oldPartitions(currentMetadataPartitions) + .newPartitions(expectPartitions) + .build()) + .dispatch()); + return updateMetadataFuture // create missing partitions .thenCompose(__ -> tryCreatePartitionsAsync(expectPartitions)) // because we should consider the compatibility. @@ -450,8 +475,7 @@ protected CompletableFuture internalCreateNonPartitionedTopicAsync(boolean .exceptionally(ex -> { Throwable rc = FutureUtil.unwrapCompletionException(ex); - if (!(rc instanceof PulsarAdminException - .ConflictException)) { + if (!(rc instanceof ConflictException)) { log.warn("[{}] got an error while copying" + " the subscription to the" + " partition {}.", topicName, @@ -666,7 +690,7 @@ private CompletableFuture internalUpdateNonPartitionedTopicProperties(Map< return; } ManagedLedger managedLedger = ((PersistentTopic) opt.get()).getManagedLedger(); - managedLedger.asyncSetProperties(properties, new AsyncCallbacks.UpdatePropertiesCallback() { + managedLedger.asyncSetProperties(properties, new UpdatePropertiesCallback() { @Override public void updatePropertiesComplete(Map properties, Object ctx) { @@ -728,7 +752,7 @@ private CompletableFuture internalRemoveNonPartitionedTopicProperties(Stri return; } ManagedLedger managedLedger = ((PersistentTopic) opt.get()).getManagedLedger(); - managedLedger.asyncDeleteProperty(key, new AsyncCallbacks.UpdatePropertiesCallback() { + managedLedger.asyncDeleteProperty(key, new UpdatePropertiesCallback() { @Override public void updatePropertiesComplete(Map properties, Object ctx) { @@ -770,8 +794,12 @@ protected CompletableFuture internalCheckNonPartitionedTopicExists(TopicNa protected void internalDeletePartitionedTopic(AsyncResponse asyncResponse, boolean authoritative, boolean force) { - validateNamespaceOperationAsync(topicName.getNamespaceObject(), NamespaceOperation.DELETE_TOPIC) - .thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) + newTopicEvent(topicName, TopicEvent.DELETE) + .stage(EventStage.BEFORE).data(TopicDeleteEventData.builder().force(force).build()) + .dispatch(); + validateTopicOwnershipAsync(topicName, authoritative) + .thenCompose(__ -> validateNamespaceOperationAsync(topicName.getNamespaceObject(), + NamespaceOperation.DELETE_TOPIC)) .thenCompose(__ -> pulsar().getBrokerService() .fetchPartitionedTopicMetadataAsync(topicName) .thenCompose(partitionedMeta -> { @@ -780,7 +808,7 @@ protected void internalDeletePartitionedTopic(AsyncResponse asyncResponse, return pulsar().getNamespaceService().checkNonPartitionedTopicExists(topicName) .thenApply(exists -> { if (exists) { - throw new RestException(Response.Status.CONFLICT, + throw new RestException(Status.CONFLICT, String.format("%s is a non-partitioned topic. Instead of calling" + " delete-partitioned-topic please call delete.", topicName)); } else { @@ -803,6 +831,8 @@ protected void internalDeletePartitionedTopic(AsyncResponse asyncResponse, .thenAccept(__ -> { log.info("[{}] Deleted partitioned topic {}", clientAppId(), topicName); asyncResponse.resume(Response.noContent().build()); + newTopicEvent(topicName, TopicEvent.DELETE) + .data(TopicDeleteEventData.builder().force(force).build()).dispatch(); }).exceptionally(ex -> { Throwable realCause = FutureUtil.unwrapCompletionException(ex); if (realCause instanceof PreconditionFailedException) { @@ -817,7 +847,7 @@ protected void internalDeletePartitionedTopic(AsyncResponse asyncResponse, getPartitionedTopicNotFoundErrorMessage(topicName.toString())))); } else if (realCause instanceof PulsarAdminException) { asyncResponse.resume(new RestException((PulsarAdminException) realCause)); - } else if (realCause instanceof MetadataStoreException.BadVersionException) { + } else if (realCause instanceof BadVersionException) { asyncResponse.resume(new RestException( new RestException(Status.CONFLICT, "Concurrent modification"))); } else { @@ -1118,12 +1148,12 @@ protected CompletableFuture internalSetDeduplicationSnapshotInterval(Integ private void internalUnloadNonPartitionedTopicAsync(AsyncResponse asyncResponse, boolean authoritative) { validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(__ -> getTopicReferenceAsync(topicName)) - .thenCompose(topic -> topic.close(false)) - .thenRun(() -> { - log.info("[{}] Successfully unloaded topic {}", clientAppId(), topicName); - asyncResponse.resume(Response.noContent().build()); - }) + .thenCompose(__ -> getTopicReferenceAsync(topicName)) + .thenCompose(topic -> topic.close(false)) + .thenRun(() -> { + log.info("[{}] Successfully unloaded topic {}", clientAppId(), topicName); + asyncResponse.resume(Response.noContent().build()); + }) .exceptionally(ex -> { // If the exception is not redirect exception we need to log it. if (isNot307And404Exception(ex)) { @@ -1667,7 +1697,14 @@ private CompletableFuture internalDeleteSubscriptionForNonPartitionedTopic throw new RestException(Status.NOT_FOUND, getSubNotFoundErrorMessage(topicName.toString(), subName)); } - return force ? sub.deleteForcefully() : sub.delete(); + CompletableFuture future = force ? sub.deleteForcefully() : sub.delete(); + return future.thenRun(() -> newTopicEvent(topicName, TopicEvent.SUBSCRIPTION_DELETE) + .data(SubscriptionDeleteEventData.builder() + .subscriptionName(subName) + .subscriptionType(sub.getType()) + .force(force) + .build()) + .dispatch()); }); } @@ -1856,7 +1893,10 @@ private CompletableFuture internalSkipAllMessagesForNonPartitionedTopicAsy String subName) { return getTopicReferenceAsync(topicName).thenCompose(t -> { PersistentTopic topic = (PersistentTopic) t; - BiConsumer biConsumer = (v, ex) -> { + AtomicReference repl = new AtomicReference<>(); + AtomicReference sub = new AtomicReference<>(); + + BiConsumer biConsumer = (cleared, ex) -> { if (ex != null) { asyncResponse.resume(new RestException(ex)); log.error("[{}] Failed to skip all messages {} {}", @@ -1864,26 +1904,48 @@ private CompletableFuture internalSkipAllMessagesForNonPartitionedTopicAsy } else { asyncResponse.resume(Response.noContent().build()); log.info("[{}] Cleared backlog on {} {}", clientAppId(), topicName, subName); + PersistentSubscription persistentSubscription = sub.get(); + SubType subscriptionType = + persistentSubscription != null ? persistentSubscription.getType() : null; + String readPosition; + if (persistentSubscription != null) { + readPosition = persistentSubscription.getCursor().getReadPosition().toString(); + } else { + PersistentReplicator persistentReplicator = repl.get(); + readPosition = persistentReplicator != null + ? persistentReplicator.getCursor().getReadPosition().toString() : null; + } + newTopicEvent(topicName, TopicEvent.SUBSCRIPTION_CLEAR_BACKLOG) + .data(SubscriptionClearBacklogEventData + .builder() + .subscriptionName(subName) + .subscriptionType(subscriptionType) + .readPosition(readPosition) + .cleared(cleared) + .build()) + .dispatch(); } }; if (subName.startsWith(topic.getReplicatorPrefix())) { String remoteCluster = PersistentReplicator.getRemoteCluster(subName); - PersistentReplicator repl = - (PersistentReplicator) topic.getPersistentReplicator(remoteCluster); - if (repl == null) { + PersistentReplicator persistentReplicator = + (PersistentReplicator) topic.getPersistentReplicator(remoteCluster); + repl.set(persistentReplicator); + if (persistentReplicator == null) { asyncResponse.resume(new RestException(Status.NOT_FOUND, getSubNotFoundErrorMessage(topicName.toString(), subName))); return CompletableFuture.completedFuture(null); } - return repl.clearBacklog().whenComplete(biConsumer); + return persistentReplicator.purgeBacklog().whenComplete(biConsumer); } else { - PersistentSubscription sub = topic.getSubscription(subName); - if (sub == null) { + PersistentSubscription persistentSubscription = topic.getSubscription(subName); + sub.set(persistentSubscription); + if (persistentSubscription == null) { asyncResponse.resume(new RestException(Status.NOT_FOUND, getSubNotFoundErrorMessage(topicName.toString(), subName))); return CompletableFuture.completedFuture(null); } - return sub.clearBacklog().whenComplete(biConsumer); + return persistentSubscription.purgeBacklog().whenComplete(biConsumer); } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. @@ -1893,6 +1955,9 @@ private CompletableFuture internalSkipAllMessagesForNonPartitionedTopicAsy } resumeAsyncResponseExceptionally(asyncResponse, ex); return null; + }) + .thenAccept(ignored -> { + // noop }); } @@ -2192,7 +2257,17 @@ private CompletableFuture internalResetCursorForNonPartitionedTopic(String throw new RestException(Status.NOT_FOUND, getSubNotFoundErrorMessage(topicName.toString(), subName)); } - return sub.resetCursor(timestamp); + return sub.resetCursorTo(timestamp) + .thenAccept((pos) -> newTopicEvent(topicName, TopicEvent.SUBSCRIPTION_SEEK) + .data(SubscriptionSeekEventData.builder() + .subscriptionName(subName) + .subscriptionType(sub.getType()) + .timestamp(timestamp) + .readPosition(pos.toString()) + .backlog(sub.getNumberOfEntriesInBacklog(false)) + .build()) + .dispatch() + ); }) .thenRun(() -> log.info("[{}][{}] Reset cursor on subscription {} to time {}", @@ -2240,8 +2315,7 @@ protected void internalCreateSubscription(AsyncResponse asyncResponse, String su // if all the partitioned failed due to // subscription-already-exist if (failureCount.incrementAndGet() == numPartitions - || !(ex instanceof PulsarAdminException - .ConflictException)) { + || !(ex instanceof ConflictException)) { partitionException.set(ex); } } @@ -2334,7 +2408,19 @@ private void internalCreateSubscriptionForNonPartitionedTopic( throw new RestException(Status.CONFLICT, "Subscription already exists for topic"); } - return topic.createSubscription(subscriptionName, InitialPosition.Latest, replicated, properties); + return topic.createSubscription(subscriptionName, InitialPosition.Latest, replicated, properties) + .thenApply((s) -> { + newTopicEvent(topicName, TopicEvent.SUBSCRIPTION_CREATE) + .data(SubscriptionCreateEventData.builder() + .subscriptionName(subscriptionName) + .durable(true) + .replicateSubscriptionState(replicated) + .subscriptionInitialPosition(InitialPosition.Latest) + .messageId(targetMessageId.toString()) + .build()) + .dispatch(); + return s; + }); }).thenCompose(subscription -> { // Mark the cursor as "inactive" as it was created without a real consumer connected ((PersistentSubscription) subscription).deactivateCursor(); @@ -2617,6 +2703,15 @@ protected void internalResetCursorOnPosition(AsyncResponse asyncResponse, String + " to position {}", clientAppId(), topicName, subName, messageId); asyncResponse.resume(Response.noContent().build()); + newTopicEvent(topicName, TopicEvent.SUBSCRIPTION_SEEK) + .data(SubscriptionSeekEventData.builder() + .subscriptionName(subName) + .subscriptionType(sub.getType()) + .messageId(messageId.toString()) + .backlog(sub.getCursor().getNumberOfEntriesInBacklog(false)) + .readPosition(seekPosition.toString()) + .build()) + .dispatch(); }).exceptionally(ex -> { Throwable t = (ex instanceof CompletionException ? ex.getCause() : ex); log.warn("[{}][{}] Failed to reset cursor on subscription {}" @@ -3220,21 +3315,21 @@ protected CompletableFuture internalGetBacklogAsync }); } - protected CompletableFuture> internalGetBacklogQuota( + protected CompletableFuture> internalGetBacklogQuota( boolean applied, boolean isGlobal) { return getTopicPoliciesAsyncWithRetry(topicName, isGlobal) .thenApply(op -> { - Map quotaMap = op + Map quotaMap = op .map(TopicPolicies::getBackLogQuotaMap) .map(map -> { - HashMap hashMap = new HashMap<>(); - map.forEach((key, value) -> hashMap.put(BacklogQuota.BacklogQuotaType.valueOf(key), value)); + HashMap hashMap = new HashMap<>(); + map.forEach((key, value) -> hashMap.put(BacklogQuotaType.valueOf(key), value)); return hashMap; }).orElse(new HashMap<>()); if (applied && quotaMap.isEmpty()) { quotaMap = getNamespacePolicies(namespaceName).backlog_quota_map; if (quotaMap.isEmpty()) { - for (BacklogQuota.BacklogQuotaType backlogQuotaType : BacklogQuota.BacklogQuotaType.values()) { + for (BacklogQuotaType backlogQuotaType : BacklogQuotaType.values()) { quotaMap.put( backlogQuotaType, namespaceBacklogQuota(namespaceName, backlogQuotaType) @@ -3299,10 +3394,10 @@ protected void internalGetBacklogSizeByMessageId(AsyncResponse asyncResponse, }); } - protected CompletableFuture internalSetBacklogQuota(BacklogQuota.BacklogQuotaType backlogQuotaType, + protected CompletableFuture internalSetBacklogQuota(BacklogQuotaType backlogQuotaType, BacklogQuotaImpl backlogQuota, boolean isGlobal) { - BacklogQuota.BacklogQuotaType finalBacklogQuotaType = backlogQuotaType == null - ? BacklogQuota.BacklogQuotaType.destination_storage : backlogQuotaType; + BacklogQuotaType finalBacklogQuotaType = backlogQuotaType == null + ? BacklogQuotaType.destination_storage : backlogQuotaType; try { // Null value means delete backlog quota. if (backlogQuota != null) { @@ -3809,7 +3904,7 @@ protected CompletableFuture internalRemoveMaxConsumers(boolean isGlobal) { } protected CompletableFuture internalTerminateAsync(boolean authoritative) { - if (SystemTopicNames.isSystemTopic(topicName)) { + if (isSystemTopic(topicName)) { return FutureUtil.failedFuture(new RestException(Status.METHOD_NOT_ALLOWED, "Termination of a system topic is not allowed")); } @@ -4404,7 +4499,7 @@ public CompletableFuture getPartitionedTopicMetadata( // If we don't do this, it can prevent namespace deletion due to inaccessible readers. authorizationFuture.thenCompose(__ -> checkLocalOrGetPeerReplicationCluster(pulsar, topicName.getNamespaceObject(), - SystemTopicNames.isSystemTopic(topicName))) + isSystemTopic(topicName))) .thenCompose(res -> pulsar.getBrokerService().fetchPartitionedTopicMetadataCheckAllowAutoCreationAsync(topicName)) .thenAccept(metadata -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ConsumerConnectEventData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ConsumerConnectEventData.java new file mode 100644 index 0000000000000..9eb1d0e92c683 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ConsumerConnectEventData.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.event.data; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.Value; +import org.apache.pulsar.broker.service.TopicEventsListener.EventData; +import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition; +import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; + +@Builder +@Value +@NoArgsConstructor(force = true) +@AllArgsConstructor +public class ConsumerConnectEventData implements EventData { + long consumerId; + String consumerName; + String address; + String subscriptionName; + boolean durable; + Boolean replicateSubscription; + SubType subscriptionType; + InitialPosition initialPosition; +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ConsumerDisconnectEventData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ConsumerDisconnectEventData.java new file mode 100644 index 0000000000000..fdc3d1fbaf9a2 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ConsumerDisconnectEventData.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.event.data; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.Value; +import org.apache.pulsar.broker.service.TopicEventsListener.EventData; +import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; + +@Builder +@Value +@NoArgsConstructor(force = true) +@AllArgsConstructor +public class ConsumerDisconnectEventData implements EventData { + long id; + String consumerName; + String address; + String subscriptionName; + SubType subscriptionType; + DisconnectInitiator initiator; +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/DisconnectInitiator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/DisconnectInitiator.java new file mode 100644 index 0000000000000..8e44f52fedc4e --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/DisconnectInitiator.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.event.data; + + +public enum DisconnectInitiator { + CLIENT, BROKER +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/LedgerPurgeEventData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/LedgerPurgeEventData.java new file mode 100644 index 0000000000000..195042d0ecbf0 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/LedgerPurgeEventData.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.event.data; + +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.Value; +import org.apache.pulsar.broker.service.TopicEventsListener.EventData; + +@Builder +@Value +@NoArgsConstructor(force = true) +@AllArgsConstructor +public class LedgerPurgeEventData implements EventData { + List ledgerInfos; + + @Builder + @Value + @NoArgsConstructor(force = true) + @AllArgsConstructor + public static class LedgerInfo { + long ledgerId; + long entries; + long timestamp; + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/LedgerRollEventData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/LedgerRollEventData.java new file mode 100644 index 0000000000000..a25c3b362195d --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/LedgerRollEventData.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.event.data; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.Value; +import org.apache.bookkeeper.mledger.ManagedLedgerEventListener.LedgerRollReason; +import org.apache.pulsar.broker.service.TopicEventsListener.EventData; + +@Builder +@Value +@NoArgsConstructor(force = true) +@AllArgsConstructor +public class LedgerRollEventData implements EventData { + long ledgerId; + LedgerRollReason reason; +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/MessageExpireEventData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/MessageExpireEventData.java new file mode 100644 index 0000000000000..4878f42088690 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/MessageExpireEventData.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.event.data; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.Value; +import org.apache.pulsar.broker.service.TopicEventsListener.EventData; + +@Builder +@Value +@NoArgsConstructor(force = true) +@AllArgsConstructor +public class MessageExpireEventData implements EventData { + String subscriptionName; + String position; +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ProducerConnectEventData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ProducerConnectEventData.java new file mode 100644 index 0000000000000..4c387c1ea39bd --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ProducerConnectEventData.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.event.data; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.Value; +import org.apache.pulsar.broker.service.TopicEventsListener.EventData; +import org.apache.pulsar.common.api.proto.ProducerAccessMode; + +@Builder +@Value +@NoArgsConstructor(force = true) +@AllArgsConstructor +public class ProducerConnectEventData implements EventData { + long producerId; + String producerName; + String address; + ProducerAccessMode accessMode; + Long epoch; +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ProducerDisconnectEventData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ProducerDisconnectEventData.java new file mode 100644 index 0000000000000..1446e76306ee2 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ProducerDisconnectEventData.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.event.data; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.Value; +import org.apache.pulsar.broker.service.TopicEventsListener.EventData; + +@Builder +@Value +@NoArgsConstructor(force = true) +@AllArgsConstructor +public class ProducerDisconnectEventData implements EventData { + long producerId; + String producerName; + String address; + DisconnectInitiator initiator; + +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ReplicatorStartEventData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ReplicatorStartEventData.java new file mode 100644 index 0000000000000..258f8180d8eaa --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ReplicatorStartEventData.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.event.data; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.Value; +import org.apache.pulsar.broker.service.TopicEventsListener.EventData; + +@Builder +@Value +@NoArgsConstructor(force = true) +@AllArgsConstructor +public class ReplicatorStartEventData implements EventData { + String replicatorId; + String localCluster; + String remoteCluster; +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ReplicatorStopEventData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ReplicatorStopEventData.java new file mode 100644 index 0000000000000..1141c606d7862 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/ReplicatorStopEventData.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.event.data; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.Value; +import org.apache.pulsar.broker.service.TopicEventsListener.EventData; + +@Builder +@Value +@NoArgsConstructor(force = true) +@AllArgsConstructor +public class ReplicatorStopEventData implements EventData { + String replicatorId; + String localCluster; + String remoteCluster; +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/SubscriptionClearBacklogEventData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/SubscriptionClearBacklogEventData.java new file mode 100644 index 0000000000000..20e463237af9c --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/SubscriptionClearBacklogEventData.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.event.data; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.Value; +import org.apache.pulsar.broker.service.TopicEventsListener.EventData; +import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; + +@Builder +@Value +@NoArgsConstructor(force = true) +@AllArgsConstructor +public class SubscriptionClearBacklogEventData implements EventData { + String subscriptionName; + SubType subscriptionType; + String readPosition; + long cleared; +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/SubscriptionCreateEventData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/SubscriptionCreateEventData.java new file mode 100644 index 0000000000000..0af7f8bab1cea --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/SubscriptionCreateEventData.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.event.data; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.Value; +import org.apache.pulsar.broker.service.TopicEventsListener.EventData; +import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition; + +@Builder +@Value +@NoArgsConstructor(force = true) +@AllArgsConstructor +public class SubscriptionCreateEventData implements EventData { + String subscriptionName; + boolean durable; + InitialPosition subscriptionInitialPosition; + String messageId; + Boolean replicateSubscriptionState; +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/SubscriptionDeleteEventData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/SubscriptionDeleteEventData.java new file mode 100644 index 0000000000000..e0d4a3aa174ec --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/SubscriptionDeleteEventData.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.event.data; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.Value; +import org.apache.pulsar.broker.service.TopicEventsListener.EventData; +import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; + +@Builder +@Value +@NoArgsConstructor(force = true) +@AllArgsConstructor +public class SubscriptionDeleteEventData implements EventData { + String subscriptionName; + SubType subscriptionType; + SubscriptionDeleteReason reason; + boolean force; + + public enum SubscriptionDeleteReason { + EXPIRED, CLIENT + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/SubscriptionSeekEventData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/SubscriptionSeekEventData.java new file mode 100644 index 0000000000000..8e4864bedb5e5 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/SubscriptionSeekEventData.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.event.data; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.Value; +import org.apache.pulsar.broker.service.TopicEventsListener.EventData; +import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; + +@Builder +@Value +@NoArgsConstructor(force = true) +@AllArgsConstructor +public class SubscriptionSeekEventData implements EventData { + String subscriptionName; + SubType subscriptionType; + String messageId; + Long timestamp; + String readPosition; + long backlog; +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicCreateEventData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicCreateEventData.java new file mode 100644 index 0000000000000..5fa2dfc3eb4d7 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicCreateEventData.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.event.data; + +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.Value; +import org.apache.pulsar.broker.service.TopicEventsListener.EventData; + +@Builder +@Value +@NoArgsConstructor(force = true) +@AllArgsConstructor +public class TopicCreateEventData implements EventData { + int partitions; + Map properties; +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicDeleteEventData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicDeleteEventData.java new file mode 100644 index 0000000000000..68e220dbd8138 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicDeleteEventData.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.event.data; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.Value; +import org.apache.pulsar.broker.service.TopicEventsListener.EventData; + +@Builder +@Value +@NoArgsConstructor(force = true) +@AllArgsConstructor +public class TopicDeleteEventData implements EventData { + boolean force; +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicLookupEventData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicLookupEventData.java new file mode 100644 index 0000000000000..7e3d516b3c7b7 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicLookupEventData.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.event.data; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.Value; +import org.apache.pulsar.broker.service.TopicEventsListener.EventData; + +@Builder +@Value +@NoArgsConstructor(force = true) +@AllArgsConstructor +public class TopicLookupEventData implements EventData { + String address; + String brokerUrl; + String brokerUrlTls; + String httpUrl; + String httpUrlTls; + boolean proxyThroughServiceUrl; + boolean authoritative; + boolean redirect; +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicMetadataUpdateEventData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicMetadataUpdateEventData.java new file mode 100644 index 0000000000000..93e6232cfed81 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicMetadataUpdateEventData.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.event.data; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.Value; +import org.apache.pulsar.broker.service.TopicEventsListener.EventData; + +@Builder +@Value +@NoArgsConstructor(force = true) +@AllArgsConstructor +public class TopicMetadataUpdateEventData implements EventData { + long oldPartitions; + long newPartitions; +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicPoliciesApplyEventData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicPoliciesApplyEventData.java new file mode 100644 index 0000000000000..10c429645d904 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicPoliciesApplyEventData.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.event.data; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.Value; +import org.apache.pulsar.broker.service.TopicEventsListener.EventData; +import org.apache.pulsar.common.policies.data.HierarchyTopicPolicies; + +@Builder +@Value +@NoArgsConstructor(force = true) +@AllArgsConstructor +public class TopicPoliciesApplyEventData implements EventData { + HierarchyTopicPolicies policies; + boolean error; +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicPoliciesUpdateEventData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicPoliciesUpdateEventData.java new file mode 100644 index 0000000000000..622f3cfb19db8 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/TopicPoliciesUpdateEventData.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.event.data; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.Value; +import org.apache.pulsar.broker.service.TopicEventsListener.EventData; + +@Builder +@Value +@NoArgsConstructor(force = true) +@AllArgsConstructor +public class TopicPoliciesUpdateEventData implements EventData { + Object policies; + String policiesClass; +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/package-info.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/package-info.java new file mode 100644 index 0000000000000..c7c0681ddf0cd --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/event/data/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.event.data; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/LookupResult.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/LookupResult.java index face01a8bffd7..0bcf2177f24bf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/LookupResult.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/LookupResult.java @@ -83,6 +83,10 @@ public LookupData getLookupData() { return lookupData; } + public Type getType() { + return type; + } + @Override public String toString() { return "LookupResult [type=" + type + ", lookupData=" + lookupData + "]"; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java index e89ae6bc99d62..165aa93e0f4c1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java @@ -34,9 +34,12 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.admin.AdminResource; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.event.data.TopicLookupEventData; import org.apache.pulsar.broker.namespace.LookupOptions; -import org.apache.pulsar.broker.web.PulsarWebResource; +import org.apache.pulsar.broker.service.ServerCnx; +import org.apache.pulsar.broker.service.TopicEventsListener.TopicEvent; import org.apache.pulsar.broker.web.RestException; import org.apache.pulsar.common.api.proto.CommandLookupTopicResponse.LookupType; import org.apache.pulsar.common.api.proto.ServerError; @@ -53,7 +56,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class TopicLookupBase extends PulsarWebResource { +public class TopicLookupBase extends AdminResource { private static final String LOOKUP_PATH_V1 = "/lookup/v2/destination/"; private static final String LOOKUP_PATH_V2 = "/lookup/v2/topic/"; @@ -130,6 +133,17 @@ protected CompletableFuture internalLookupTopicAsync(final TopicName if (log.isDebugEnabled()) { log.debug("Redirect lookup for topic {} to {}", topicName, redirect); } + var eventDataBuilder = + TopicLookupEventData.builder().authoritative(newAuthoritative); + if (isRequestHttps()) { + eventDataBuilder.httpUrlTls(redirect.toString()); + } else { + eventDataBuilder.httpUrl(redirect.toString()); + } + eventDataBuilder.address(formatHttpAddress()); + newTopicEvent(topicName, TopicEvent.LOOKUP) + .data(eventDataBuilder.build()) + .dispatch(); throw new WebApplicationException(Response.temporaryRedirect(redirect).build()); } else { // Found broker owning the topic @@ -138,6 +152,17 @@ protected CompletableFuture internalLookupTopicAsync(final TopicName result.getLookupData()); } pulsar().getBrokerService().getLookupRequestSemaphore().release(); + newTopicEvent(topicName, TopicEvent.LOOKUP) + .data(TopicLookupEventData.builder() + .authoritative(authoritative) + .redirect(false) + .httpUrl(result.getLookupData().getHttpUrl()) + .httpUrlTls(result.getLookupData().getHttpUrlTls()) + .brokerUrl(result.getLookupData().getBrokerUrl()) + .brokerUrlTls(result.getLookupData().getBrokerUrlTls()) + .address(formatHttpAddress()) + .build()) + .dispatch(); return result.getLookupData(); } }); @@ -147,6 +172,24 @@ protected CompletableFuture internalLookupTopicAsync(final TopicName }); } + private String formatHttpAddress() { + if (httpRequest == null) { + return "[L:/-:-1 - R:/-:-1]"; + } + + String id = "0x" + Integer.toHexString(System.identityHashCode(httpRequest)); + + String localAddr = httpRequest.getLocalAddr(); + int localPort = httpRequest.getLocalPort(); + String remoteAddr = httpRequest.getRemoteAddr(); + int remotePort = httpRequest.getRemotePort(); + + return "[id: " + id + ", L:/" + localAddr + ':' + localPort + + " - R:/" + remoteAddr + ':' + remotePort + + "]"; + } + + protected String internalGetNamespaceBundle(TopicName topicName) { validateNamespaceOperation(topicName.getNamespaceObject(), NamespaceOperation.GET_BUNDLE); try { @@ -186,7 +229,7 @@ public static CompletableFuture lookupTopicAsync(PulsarService pulsarSe AuthenticationDataSource authenticationData, AuthenticationDataSource originalAuthenticationData, long requestId, final String advertisedListenerName, - Map properties) { + Map properties, ServerCnx cnx) { final CompletableFuture validationFuture = new CompletableFuture<>(); final CompletableFuture lookupfuture = new CompletableFuture<>(); @@ -201,6 +244,21 @@ public static CompletableFuture lookupTopicAsync(PulsarService pulsarSe differentClusterData.getBrokerServiceUrl(), differentClusterData.getBrokerServiceUrlTls(), cluster); } + if (cnx != null) { + cnx.newTopicEvent(topicName.toString(), TopicEvent.LOOKUP) + .data(TopicLookupEventData.builder() + .authoritative(authoritative) + .httpUrl(differentClusterData.getBrokerServiceUrl()) + .httpUrlTls(differentClusterData.getBrokerServiceUrlTls()) + .brokerUrl(differentClusterData.getBrokerServiceUrl()) + .brokerUrlTls(differentClusterData.getBrokerServiceUrlTls()) + .authoritative(true) + .redirect(true) + .proxyThroughServiceUrl(false) + .address(cnx.toString()) + .build()) + .dispatch(); + } validationFuture.complete(newLookupResponse(differentClusterData.getBrokerServiceUrl(), differentClusterData.getBrokerServiceUrlTls(), true, LookupType.Redirect, requestId, false)); @@ -295,16 +353,43 @@ public static CompletableFuture lookupTopicAsync(PulsarService pulsarSe LookupData lookupData = lookupResult.get().getLookupData(); printWarnLogIfLookupResUnexpected(topicName, lookupData, options, pulsarService); + boolean proxyThroughServiceUrl; + boolean newAuthoritative; if (lookupResult.get().isRedirect()) { - boolean newAuthoritative = lookupResult.get().isAuthoritativeRedirect(); + proxyThroughServiceUrl = false; + newAuthoritative = lookupResult.get().isAuthoritativeRedirect(); lookupfuture.complete( newLookupResponse(lookupData.getBrokerUrl(), lookupData.getBrokerUrlTls(), - newAuthoritative, LookupType.Redirect, requestId, false)); + newAuthoritative, LookupType.Redirect, requestId, + proxyThroughServiceUrl)); } else { ServiceConfiguration conf = pulsarService.getConfiguration(); + proxyThroughServiceUrl = shouldRedirectThroughServiceUrl(conf, lookupData); + newAuthoritative = true; lookupfuture.complete(newLookupResponse(lookupData.getBrokerUrl(), - lookupData.getBrokerUrlTls(), true /* authoritative */, LookupType.Connect, - requestId, shouldRedirectThroughServiceUrl(conf, lookupData))); + lookupData.getBrokerUrlTls(), newAuthoritative /* authoritative */, + LookupType.Connect, + requestId, proxyThroughServiceUrl)); + } + if (cnx != null) { + lookupfuture.whenComplete((__, ex) -> { + if (ex == null) { + cnx.newTopicEvent(topicName.toString(), TopicEvent.LOOKUP) + .clientVersion(cnx.getClientVersion()) + .proxyVersion(cnx.getProxyVersion()) + .data(TopicLookupEventData.builder() + .authoritative(newAuthoritative) + .httpUrl(lookupData.getHttpUrl()) + .httpUrlTls(lookupData.getHttpUrlTls()) + .brokerUrl(lookupData.getBrokerUrl()) + .brokerUrlTls(lookupData.getBrokerUrlTls()) + .redirect(lookupResult.get().isRedirect()) + .proxyThroughServiceUrl(proxyThroughServiceUrl) + .address(cnx.toString()) + .build()) + .dispatch(); + } + }); } }).exceptionally(ex -> { handleLookupError(lookupfuture, topicName.toString(), clientAppId, requestId, ex); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java index aa642eb79a4e6..f11a839f8bf0c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java @@ -692,8 +692,8 @@ protected static double getRgQuotaMessage(String rgName, String monClassName, St @VisibleForTesting protected static long getRgLocalUsageByteCount(String rgName, String monClassName, String localCluster, String remoteCluster) { - return (long) rgLocalUsageBytes.labels(rgName, monClassName, localCluster, remoteCluster != null ? remoteCluster : "") - .get(); + return (long) rgLocalUsageBytes.labels(rgName, monClassName, localCluster, + remoteCluster != null ? remoteCluster : "").get(); } // Visibility for testing. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java index f996d328090ca..e86c84feb2da3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java @@ -30,8 +30,10 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.event.data.ReplicatorStopEventData; import org.apache.pulsar.broker.service.BrokerServiceException.NamingException; import org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException; +import org.apache.pulsar.broker.service.TopicEventsListener.TopicEvent; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; @@ -437,6 +439,16 @@ public CompletableFuture terminate() { disableReplicatorRead(); // release resources. doReleaseResources(); + TopicEventsDispatcher topicEventsDispatcher = brokerService.getTopicEventsDispatcher(); + if (topicEventsDispatcher != null) { + topicEventsDispatcher.newEvent(localTopicName, TopicEvent.REPLICATOR_STOP) + .data(ReplicatorStopEventData.builder() + .replicatorId(replicatorId) + .localCluster(localCluster) + .remoteCluster(remoteCluster) + .build()) + .dispatch(); + } }); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index c8018b8c3f12b..185c8b9fff93d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -109,6 +109,8 @@ import org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerFactory; import org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerLoader; import org.apache.pulsar.broker.delayed.InMemoryDelayedDeliveryTrackerFactory; +import org.apache.pulsar.broker.event.data.TopicCreateEventData; +import org.apache.pulsar.broker.event.data.TopicDeleteEventData; import org.apache.pulsar.broker.intercept.BrokerInterceptor; import org.apache.pulsar.broker.intercept.ManagedLedgerInterceptorImpl; import org.apache.pulsar.broker.loadbalance.LoadManager; @@ -124,6 +126,7 @@ import org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException; import org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException; import org.apache.pulsar.broker.service.BrokerServiceException.TopicMigratedException; +import org.apache.pulsar.broker.service.TopicEventsDispatcher.TopicEventBuilder; import org.apache.pulsar.broker.service.TopicEventsListener.EventStage; import org.apache.pulsar.broker.service.TopicEventsListener.TopicEvent; import org.apache.pulsar.broker.service.nonpersistent.NonPersistentSystemTopic; @@ -337,12 +340,13 @@ public class BrokerService implements Closeable { private Set brokerEntryMetadataInterceptors; private Set brokerEntryPayloadProcessors; - private final TopicEventsDispatcher topicEventsDispatcher = new TopicEventsDispatcher(); + private final TopicEventsDispatcher topicEventsDispatcher; private volatile boolean unloaded = false; public BrokerService(PulsarService pulsar, EventLoopGroup eventLoopGroup) throws Exception { this.pulsar = pulsar; this.clock = pulsar.getClock(); + this.topicEventsDispatcher = new TopicEventsDispatcher(pulsar); this.dynamicConfigurationMap = prepareDynamicConfigurationMap(); this.brokerPublishRateLimiter = new PublishRateLimiterImpl(pulsar.getMonotonicClock(), producer -> { producer.getCnx().getThrottleTracker().markThrottled( @@ -507,7 +511,7 @@ private int getPendingTopicLoadRequests() { public void addTopicEventListener(TopicEventsListener... listeners) { topicEventsDispatcher.addTopicEventListener(listeners); topics.keySet().forEach(topic -> - TopicEventsDispatcher.notify(listeners, topic, TopicEvent.LOAD, EventStage.SUCCESS, null)); + topicEventsDispatcher.newEvent(topic, TopicEvent.LOAD).dispatch(listeners)); } public void removeTopicEventListener(TopicEventsListener... listeners) { @@ -1066,12 +1070,22 @@ public CompletableFuture> getTopicIfExists(final String topic) { return getTopic(topic, false /* createIfMissing */); } + @Deprecated public CompletableFuture getOrCreateTopic(final String topic) { return isAllowAutoTopicCreationAsync(topic) .thenCompose(isAllowed -> getTopic(topic, isAllowed)) .thenApply(Optional::get); } + public CompletableFuture getOrCreateTopic(TopicLoadingContext context) { + return isAllowAutoTopicCreationAsync(context.getTopicName()) + .thenCompose(isAllowed -> { + context.setCreateIfMissing(isAllowed); + return getTopic(context); + }) + .thenApply(Optional::get); + } + public CompletableFuture> getTopic(final String topic, boolean createIfMissing) { return getTopic(topic, createIfMissing, null); } @@ -1160,9 +1174,29 @@ private CompletableFuture validateTopicConsistency(TopicName topicName) { * @param properties Topic configuration properties used during creation. * @return CompletableFuture with an Optional of the topic if found or created, otherwise empty. */ + @Deprecated public CompletableFuture> getTopic(final TopicName topicName, boolean createIfMissing, - @Nullable Map properties) { + Map properties) { + return getTopic(TopicLoadingContext.builder().topicName(topicName).createIfMissing(createIfMissing) + .properties(properties).build()); + } + + /** + * Retrieves or creates a topic based on the specified parameters. + * 0. If disable PersistentTopics or NonPersistentTopics, it will return a failed future with NotAllowedException. + * 1. If topic future exists in the cache returned directly regardless of whether it fails or timeout. + * 2. If the topic metadata exists, the topic is created regardless of {@code createIfMissing}. + * 3. If the topic metadata not exists, and {@code createIfMissing} is false, + * returns an empty Optional in a CompletableFuture. And this empty future not be added to the map. + * 4. Otherwise, use computeIfAbsent. It returns the existing topic or creates and adds a new topicFuture. + * Any exceptions will remove the topicFuture from the map. + * + * @return CompletableFuture with an Optional of the topic if found or created, otherwise empty. + */ + public CompletableFuture> getTopic(TopicLoadingContext context) { + TopicName topicName = context.getTopicName(); try { + var createIfMissing = context.isCreateIfMissing(); // If topic future exists in the cache returned directly regardless of whether it fails or timeout. CompletableFuture> tp = topics.get(topicName.toString()); if (tp != null) { @@ -1177,13 +1211,13 @@ public CompletableFuture> getTopic(final TopicName topicName, bo return FutureUtil.failedFuture(new NotAllowedException( "Broker is unable to load persistent topic")); } - final CompletableFuture> topicFuture = FutureUtil.createFutureWithTimeout( - Duration.ofSeconds(pulsar.getConfiguration().getTopicLoadTimeoutSeconds()), executor(), - () -> FAILED_TO_LOAD_TOPIC_TIMEOUT_EXCEPTION); - final var context = new TopicLoadingContext(topicName, createIfMissing, topicFuture); - if (properties != null) { - context.setProperties(properties); + if (context.getTopicFuture() == null) { + final CompletableFuture> topicFuture = FutureUtil.createFutureWithTimeout( + Duration.ofSeconds(pulsar.getConfiguration().getTopicLoadTimeoutSeconds()), executor(), + () -> FAILED_TO_LOAD_TOPIC_TIMEOUT_EXCEPTION); + context.setTopicFuture(topicFuture); } + var topicFuture = context.getTopicFuture(); topicFuture.exceptionally(t -> { final var now = System.nanoTime(); if (FutureUtil.unwrapCompletionException(t) instanceof TimeoutException) { @@ -1206,7 +1240,17 @@ public CompletableFuture> getTopic(final TopicName topicName, bo final var inserted = new MutableBoolean(false); final var cachedFuture = topics.computeIfAbsent(topicName.toString(), ___ -> { inserted.setTrue(); - return loadOrCreatePersistentTopic(context); + var loadEventBuild = topicEventsDispatcher.newEvent(topicName.toString(), TopicEvent.LOAD) + .clientVersion(context.getClientVersion()) + .proxyVersion(context.getProxyVersion()) + .stage(EventStage.BEFORE); + loadEventBuild.dispatch(); + CompletableFuture> future = loadOrCreatePersistentTopic(context); + future.whenComplete((topic, ex) -> { + loadEventBuild.stage(ex != null ? EventStage.FAILURE : EventStage.SUCCESS) + .error(ex).dispatch(); + }); + return future; }); if (inserted.isFalse()) { // This case should happen rarely when the same topic is loaded concurrently because we @@ -1242,29 +1286,25 @@ public CompletableFuture> getTopic(final TopicName topicName, bo return FutureUtil.failedFuture(new NotAllowedException( "Broker is unable to load persistent topic")); } - if (!topics.containsKey(topicName.toString())) { - topicEventsDispatcher.notify(topicName.toString(), TopicEvent.LOAD, EventStage.BEFORE); + + CompletableFuture> existing = topics.get(topicName.toString()); + if (existing != null) { + return existing; } - if (topicName.isPartitioned() || createIfMissing) { - return topics.computeIfAbsent(topicName.toString(), (name) -> { - topicEventsDispatcher - .notify(topicName.toString(), TopicEvent.CREATE, EventStage.BEFORE); - CompletableFuture> res = createNonPersistentTopic(name); + newTopicEvent(context, TopicEvent.LOAD) + .stage(EventStage.BEFORE) + .dispatch(); - CompletableFuture> eventFuture = topicEventsDispatcher - .notifyOnCompletion(res, topicName.toString(), TopicEvent.CREATE); - topicEventsDispatcher - .notifyOnCompletion(eventFuture, topicName.toString(), TopicEvent.LOAD); - return res; - }); - } - CompletableFuture> topicFuture = topics.get(topicName.toString()); - if (topicFuture == null) { - topicEventsDispatcher.notify(topicName.toString(), TopicEvent.LOAD, EventStage.FAILURE); - topicFuture = CompletableFuture.completedFuture(Optional.empty()); + if (topicName.isPartitioned() || createIfMissing) { + return topics.computeIfAbsent(topicName.toString(), (name) -> createNonPersistentTopic(context)); } - return topicFuture; + + newTopicEvent(context, TopicEvent.LOAD) + .stage(EventStage.FAILURE) + .dispatch(); + + return CompletableFuture.completedFuture(Optional.empty()); } } catch (IllegalArgumentException e) { log.warn("[{}] Illegalargument exception when loading topic", topicName, e); @@ -1289,10 +1329,27 @@ private CompletableFuture> getTopicPoliciesBypassSystemT return pulsar.getTopicPoliciesService().getTopicPoliciesAsync(topicName, type); } + private TopicEventsDispatcher.TopicEventBuilder newTopicEvent(TopicLoadingContext topicLoadingContext, + TopicEvent topicEvent) { + return getTopicEventsDispatcher().newEvent(topicLoadingContext.getTopicName().toString(), topicEvent) + .clientVersion(topicLoadingContext.getClientVersion()) + .proxyVersion(topicLoadingContext.getProxyVersion()); + } + public CompletableFuture deleteTopic(String topic, boolean forceDelete) { - topicEventsDispatcher.notify(topic, TopicEvent.DELETE, EventStage.BEFORE); - CompletableFuture result = deleteTopicInternal(topic, forceDelete); - topicEventsDispatcher.notifyOnCompletion(result, topic, TopicEvent.DELETE); + TopicDeleteEventData deleteEventData = TopicDeleteEventData.builder().force(forceDelete).build(); + topicEventsDispatcher.newEvent(topic, TopicEvent.DELETE) + .data(deleteEventData) + .stage(EventStage.BEFORE) + .dispatch(); + CompletableFuture result = deleteTopicInternal(topic, forceDelete); + result.whenComplete((__, ex) -> { + topicEventsDispatcher.newEvent(topic, TopicEvent.DELETE) + .data(deleteEventData) + .stage(ex != null ? EventStage.FAILURE : EventStage.SUCCESS) + .error(ex) + .dispatch(); + }); return result; } @@ -1423,7 +1480,32 @@ public void deleteTopicAuthenticationWithRetry(String topic, CompletableFuture> createNonPersistentTopic(String topic) { + private CompletableFuture> createNonPersistentTopic(TopicLoadingContext context) { + String topic = context.getTopicName().toString(); + TopicName topicName = context.getTopicName(); + TopicCreateEventData topicCreateEventData = TopicCreateEventData + .builder() + .properties(context.getProperties()) + .build(); + // For partitioned topic, the creation event is already sent by the admin API. + // For non-partitioned topic, send the creation event here instead. + boolean sendCreationEvent = !topicName.isPartitioned(); + + Consumer dispatchEvent = (ex) -> { + if (sendCreationEvent) { + newTopicEvent(context, TopicEvent.CREATE).data(topicCreateEventData) + .stage(ex != null ? EventStage.FAILURE : EventStage.SUCCESS).error(ex).dispatch(); + } + newTopicEvent(context, TopicEvent.LOAD).stage(ex != null ? EventStage.FAILURE : EventStage.SUCCESS) + .error(ex).dispatch(); + }; + + if (sendCreationEvent) { + newTopicEvent(context, TopicEvent.CREATE) + .stage(EventStage.BEFORE) + .data(topicCreateEventData) + .dispatch(); + } CompletableFuture> topicFuture = new CompletableFuture<>(); topicFuture.exceptionally(t -> { pulsarStats.recordTopicLoadFailed(); @@ -1447,25 +1529,43 @@ private CompletableFuture> createNonPersistentTopic(String topic checkTopicNsOwnership(topic) .thenCompose((__) -> validateTopicConsistency(TopicName.get(topic))) .thenRun(() -> { - nonPersistentTopic.initialize() - .thenCompose(__ -> nonPersistentTopic.checkReplication()) - .thenRun(() -> { - log.info("Created topic {}", nonPersistentTopic); - long topicLoadLatencyMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()) - topicCreateTimeMs; - pulsarStats.recordTopicLoadTimeValue(topic, topicLoadLatencyMs); - addTopicToStatsMaps(TopicName.get(topic), nonPersistentTopic); - topicFuture.complete(Optional.of(nonPersistentTopic)); - }).exceptionally(ex -> { - log.warn("Replication check failed. Removing topic from topics list {}, {}", topic, ex.getCause()); - nonPersistentTopic.stopReplProducers().whenComplete((v, exception) -> { - topicFuture.completeExceptionally(ex); + nonPersistentTopic.initialize() + .thenCompose(__ -> nonPersistentTopic.checkReplication()) + .thenRun(() -> { + log.info("Created topic {}", nonPersistentTopic); + long topicLoadLatencyMs = + TimeUnit.NANOSECONDS.toMillis(System.nanoTime()) - topicCreateTimeMs; + pulsarStats.recordTopicLoadTimeValue(topic, topicLoadLatencyMs); + addTopicToStatsMaps(TopicName.get(topic), nonPersistentTopic); + + dispatchEvent.accept(null); + topicFuture.complete(Optional.of(nonPersistentTopic)); + }).exceptionally(ex -> { + log.warn("Replication check failed. Removing topic from topics list {}, {}", topic, + ex.getCause()); + nonPersistentTopic.stopReplProducers().whenComplete((v, exception) -> { + dispatchEvent.accept(ex); + topicFuture.completeExceptionally(ex); + }); + return null; + }); + }).exceptionally(e -> { + log.warn("CheckTopicNsOwnership fail when createNonPersistentTopic! {}", topic, e.getCause()); + dispatchEvent.accept(null); + // CheckTopicNsOwnership fail dont create nonPersistentTopic, when topic do lookup will find the + // correct + // broker. When client get non-persistent-partitioned topic + // metadata will the non-persistent-topic will be created. + // so we should add checkTopicNsOwnership logic otherwise the topic will be created + // if it dont own by this broker,we should return success + // otherwise it will keep retrying getPartitionedTopicMetadata + topicFuture.complete(Optional.of(nonPersistentTopic)); + // after get metadata return success, we should delete this topic from this broker, because this + // topic not + // owner by this broker and it don't initialize and checkReplication + pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); + return null; }); - return null; - }); - }).exceptionally(e -> { - topicFuture.completeExceptionally(FutureUtil.unwrapCompletionException(e)); - return null; - }); return topicFuture; } @@ -1851,25 +1951,19 @@ public void createPersistentTopic0(TopicLoadingContext context) { managedLedgerConfig.setShadowSourceName(TopicName.get(shadowSource).getPersistenceNamingEncoding()); } - topicEventsDispatcher.notify(topic, TopicEvent.LOAD, EventStage.BEFORE); - // load can fail with topicFuture completed non-exceptionally - // work around this - final CompletableFuture loadFuture = new CompletableFuture<>(); - topicFuture.whenComplete((res, ex) -> { - if (ex == null) { - loadFuture.complete(null); - } else { - loadFuture.completeExceptionally(ex); - } - }); - - if (createIfMissing) { - if (!exists) { - topicEventsDispatcher.notify(topic, TopicEvent.CREATE, EventStage.BEFORE); - topicEventsDispatcher.notifyOnCompletion(topicFuture, topic, TopicEvent.CREATE); - } + TopicEventBuilder createEventBuild; + if (createIfMissing && !exists) { + createEventBuild = topicEventsDispatcher.newEvent(topic, TopicEvent.CREATE) + .clientVersion(context.getClientVersion()) + .proxyVersion(context.getProxyVersion()) + .stage(EventStage.BEFORE) + .data(TopicCreateEventData.builder() + .properties(context.getProperties()) + .build()); + createEventBuild.dispatch(); + } else { + createEventBuild = null; } - topicEventsDispatcher.notifyOnCompletion(loadFuture, topic, TopicEvent.LOAD); // Once we have the configuration, we can proceed with the async open operation ManagedLedgerFactory managedLedgerFactory = @@ -1898,11 +1992,18 @@ public void openLedgerComplete(ManagedLedger ledger, Object ctx) { persistentTopic.isDeduplicationEnabled() ? "enabled" : "disabled", context.latencyString(nowInNanos)); pulsarStats.recordTopicLoadTimeValue(topic, topicLoadLatencyMs); + if (createEventBuild != null) { + createEventBuild + .stage(EventStage.SUCCESS) + .dispatch(); + } if (!topicFuture.complete(Optional.of(persistentTopic))) { // Check create persistent topic timeout. if (topicFuture.isCompletedExceptionally()) { + Optional exception = + FutureUtil.getException(topicFuture); log.warn("{} future is already completed with failure {}, closing" - + " the topic", topic, FutureUtil.getException(topicFuture)); + + " the topic", topic, exception); } else { // It should not happen. log.error("{} future is already completed by another thread, " @@ -1922,6 +2023,11 @@ public void openLedgerComplete(ManagedLedger ledger, Object ctx) { } }) .exceptionally((ex) -> { + if (createEventBuild != null) { + createEventBuild + .stage(EventStage.FAILURE) + .dispatch(); + } log.warn("Replication or dedup check failed." + " Removing topic from topics list {}, {}", topic, ex); executor().submit(() -> { @@ -1937,6 +2043,12 @@ public void openLedgerComplete(ManagedLedger ledger, Object ctx) { return null; }); } catch (Exception e) { + if (createEventBuild != null) { + createEventBuild + .stage(EventStage.FAILURE) + .error(e) + .dispatch(); + } log.warn("Failed to create topic {}: {}", topic, e.getMessage()); pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); topicFuture.completeExceptionally(e); @@ -1948,7 +2060,6 @@ public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { if (!createIfMissing && exception instanceof ManagedLedgerNotFoundException) { // We were just trying to load a topic and the topic doesn't exist pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); - loadFuture.completeExceptionally(exception); topicFuture.complete(Optional.empty()); } else { log.warn("Failed to create topic {}", topic, exception); @@ -2316,15 +2427,15 @@ public void checkCompaction() { }); } - private void checkConsumedLedgers() { + @VisibleForTesting + public void checkConsumedLedgers() { forEachTopic((t) -> { if (t instanceof PersistentTopic) { Optional.ofNullable(((PersistentTopic) t).getManagedLedger()).ifPresent( managedLedger -> { managedLedger.trimConsumedLedgersInBackground(Futures.NULL_PROMISE); managedLedger.rolloverCursorsInBackground(); - } - ); + }); } }); } @@ -2540,7 +2651,7 @@ private void removeTopicFromCache(String topic, NamespaceBundle namespaceBundle, String bundleName = namespaceBundle.toString(); String namespaceName = TopicName.get(topic).getNamespaceObject().toString(); - topicEventsDispatcher.notify(topic, TopicEvent.UNLOAD, EventStage.BEFORE); + topicEventsDispatcher.newEvent(topic, TopicEvent.UNLOAD).stage(EventStage.BEFORE).dispatch(); synchronized (multiLayerTopicsMap) { final var namespaceMap = multiLayerTopicsMap.get(namespaceName); @@ -2575,7 +2686,7 @@ private void removeTopicFromCache(String topic, NamespaceBundle namespaceBundle, if (compactor != null) { compactor.getStats().removeTopic(topic); } - topicEventsDispatcher.notify(topic, TopicEvent.UNLOAD, EventStage.SUCCESS); + topicEventsDispatcher.newEvent(topic, TopicEvent.UNLOAD).dispatch(); } public long getNumberOfNamespaceBundles() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 1038baf6ba23d..51a57c1071a93 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -490,8 +490,8 @@ public void disconnect(boolean isResetCursor, Optional assigne } } - public void doUnsubscribe(final long requestId, boolean force) { - subscription.doUnsubscribe(this, force).thenAccept(v -> { + public CompletableFuture doUnsubscribe(final long requestId, boolean force) { + return subscription.doUnsubscribe(this, force).thenAccept(v -> { log.info("Unsubscribed successfully from {}", subscription); cnx.removedConsumer(this); cnx.getCommandSender().sendSuccessResponse(requestId); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index cada292dbca70..d96e72df3ed94 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -41,6 +41,7 @@ import io.netty.channel.ChannelOutboundBuffer; import io.netty.handler.codec.haproxy.HAProxyMessage; import io.netty.handler.ssl.SslHandler; +import io.netty.util.AttributeKey; import io.netty.util.concurrent.FastThreadLocal; import io.netty.util.concurrent.Promise; import io.netty.util.concurrent.ScheduledFuture; @@ -86,6 +87,15 @@ import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription; import org.apache.pulsar.broker.authentication.AuthenticationProvider; import org.apache.pulsar.broker.authentication.AuthenticationState; +import org.apache.pulsar.broker.event.data.ConsumerConnectEventData; +import org.apache.pulsar.broker.event.data.ConsumerDisconnectEventData; +import org.apache.pulsar.broker.event.data.DisconnectInitiator; +import org.apache.pulsar.broker.event.data.ProducerConnectEventData; +import org.apache.pulsar.broker.event.data.ProducerConnectEventData.ProducerConnectEventDataBuilder; +import org.apache.pulsar.broker.event.data.ProducerDisconnectEventData; +import org.apache.pulsar.broker.event.data.SubscriptionDeleteEventData; +import org.apache.pulsar.broker.event.data.SubscriptionDeleteEventData.SubscriptionDeleteReason; +import org.apache.pulsar.broker.event.data.SubscriptionSeekEventData; import org.apache.pulsar.broker.intercept.BrokerInterceptor; import org.apache.pulsar.broker.limiter.ConnectionController; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; @@ -97,6 +107,7 @@ import org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException; import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionNotFoundException; import org.apache.pulsar.broker.service.BrokerServiceException.TopicNotFoundException; +import org.apache.pulsar.broker.service.TopicEventsListener.TopicEvent; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.service.schema.SchemaRegistryService; @@ -271,6 +282,9 @@ public class ServerCnx extends PulsarHandler implements TransportCnx { private boolean pausedDueToRateLimitation = false; private AsyncDualMemoryLimiterImpl maxTopicListInFlightLimiter; + private static final AttributeKey CLOSED_BY_LOCAL_ATTR_KEY = + AttributeKey.valueOf("CLOSED_BY_LOCAL"); + // Tracks and limits number of bytes pending to be published from a single specific IO thread. static final class PendingBytesPerThreadTracker { private static final FastThreadLocal pendingBytesPerThread = @@ -429,6 +443,8 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { authRefreshTask.cancel(false); } + boolean closeByBroker = Boolean.TRUE.equals(ctx.channel().attr(CLOSED_BY_LOCAL_ATTR_KEY).get()); + // Connection is gone, close the producers immediately producers.forEach((__, producerFuture) -> { // prevent race conditions in completing producers @@ -442,6 +458,8 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { if (brokerInterceptor != null) { brokerInterceptor.producerClosed(this, producer, producer.getMetadata()); } + dispatchCloseProducerEvent(producer, + closeByBroker ? DisconnectInitiator.BROKER : DisconnectInitiator.CLIENT); } }); @@ -458,6 +476,8 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { if (brokerInterceptor != null) { brokerInterceptor.consumerClosed(this, consumer, consumer.getMetadata()); } + dispatchCloseConsumerEvent(consumer, + closeByBroker ? DisconnectInitiator.BROKER : DisconnectInitiator.CLIENT); } catch (BrokerServiceException e) { log.warn("Consumer {} was already closed: {}", consumer, e); } @@ -639,7 +659,7 @@ protected void handleLookup(CommandLookupTopic lookupParam) { lookupTopicAsync(getBrokerService().pulsar(), topicName, authoritative, authRole, originalPrincipal, authenticationData, originalAuthData != null ? originalAuthData : authenticationData, - requestId, advertisedListenerName, properties).handle((lookupResponse, ex) -> { + requestId, advertisedListenerName, properties, this).handle((lookupResponse, ex) -> { if (ex == null) { writeAndFlush(lookupResponse); } else { @@ -1413,8 +1433,12 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { service.isAllowAutoTopicCreationAsync(topicName.toString()) .thenApply(isAllowed -> forceTopicCreation && isAllowed) - .thenCompose(createTopicIfDoesNotExist -> - service.getTopic(topicName.toString(), createTopicIfDoesNotExist)) + .thenCompose(createTopicIfDoesNotExist -> service.getTopic(TopicLoadingContext.builder() + .topicName(topicName) + .createIfMissing(createTopicIfDoesNotExist) + .clientVersion(clientVersion) + .proxyVersion(proxyVersion) + .build())) .thenCompose(optTopic -> { if (!optTopic.isPresent()) { return FutureUtil @@ -1498,6 +1522,18 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { log.info("[{}] Created subscription on topic {} / {}", remoteAddress, topicName, subscriptionName); commandSender.sendSuccessResponse(requestId); + newTopicEvent(topicName.toString(), TopicEvent.CONSUMER_CONNECT) + .data(ConsumerConnectEventData.builder() + .consumerId(consumer.consumerId()) + .consumerName(consumer.consumerName()) + .address(consumer.cnx().toString()) + .subscriptionType(subType) + .subscriptionName(subscriptionName) + .replicateSubscription(isReplicated) + .initialPosition(initialPosition) + .durable(isDurable) + .build()) + .dispatch(); if (brokerInterceptor != null) { try { brokerInterceptor.consumerCreated(this, consumer, metadata); @@ -1545,7 +1581,10 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { topicName, remoteAddress, consumerId); } consumers.remove(consumerId, consumerFuture); - closeConsumer(consumerId, Optional.empty()); + dispatchCloseConsumerEvent(topicName.toString(), consumerId, consumerName, + this.clientSourceAddress(), subscriptionName, subType, + DisconnectInitiator.BROKER); + sendCloseConsumer(consumerId, Optional.empty()); return null; } } else if (exception.getCause() instanceof BrokerServiceException) { @@ -1685,7 +1724,11 @@ protected void handleProducer(final CommandProducer cmdProducer) { topicName, producerId, producerName, schema == null ? "absent" : "present"); } - service.getOrCreateTopic(topicName.toString()).thenCompose((Topic topic) -> { + service.getOrCreateTopic(TopicLoadingContext.builder() + .topicName(topicName) + .clientVersion(clientVersion) + .proxyVersion(proxyVersion) + .build()).thenCompose((Topic topic) -> { // Check max producer limitation to avoid unnecessary ops wasting resources. For example: the new // producer reached max producer limitation, but pulsar did schema check first, it would waste CPU if (((AbstractTopic) topic).isProducersExceeded(producerName)) { @@ -1830,7 +1873,9 @@ protected void handleProducer(final CommandProducer cmdProducer) { remoteAddress, producerId); } producers.remove(producerId, producerFuture); - closeProducer(producerId, -1L, Optional.empty()); + dispatchCloseProducerEvent(topicName.toString(), producerId, producerName, + this.clientSourceAddress(), DisconnectInitiator.BROKER); + sendCloseProducer(producerId, -1L, Optional.empty()); return null; } } @@ -1883,6 +1928,15 @@ private void buildProducerAndAddTopic(Topic topic, long producerId, String produ commandSender.sendProducerSuccessResponse(requestId, producerName, producer.getLastSequenceId(), producer.getSchemaVersion(), newTopicEpoch, true /* producer is ready now */); + ProducerConnectEventDataBuilder producerConnectEventDataBuilder = ProducerConnectEventData.builder() + .accessMode(producerAccessMode) + .address(producer.getCnx().toString()) + .producerId(producer.getProducerId()) + .producerName(producer.getProducerName()); + newTopicEpoch.ifPresent(producerConnectEventDataBuilder::epoch); + newTopicEvent(topic.getName(), TopicEvent.PRODUCER_CONNECT) + .data(producerConnectEventDataBuilder.build()) + .dispatch(); if (brokerInterceptor != null) { try { brokerInterceptor.producerCreated(this, producer, metadata); @@ -2171,7 +2225,19 @@ protected void handleUnsubscribe(CommandUnsubscribe unsubscribe) { CompletableFuture consumerFuture = consumers.get(unsubscribe.getConsumerId()); if (consumerFuture != null && consumerFuture.isDone() && !consumerFuture.isCompletedExceptionally()) { - consumerFuture.getNow(null).doUnsubscribe(unsubscribe.getRequestId(), unsubscribe.isForce()); + Consumer consumer = consumerFuture.getNow(null); + String topicName = consumer.getSubscription().getTopicName(); + String subscriptionName = consumer.getSubscription().getName(); + SubType subscriptionType = consumer.getSubscription().getType(); + consumer.doUnsubscribe(unsubscribe.getRequestId(), unsubscribe.isForce()).thenRun(() -> { + newTopicEvent(topicName, TopicEvent.SUBSCRIPTION_DELETE) + .data(SubscriptionDeleteEventData.builder() + .subscriptionName(subscriptionName) + .subscriptionType(subscriptionType) + .reason(SubscriptionDeleteReason.CLIENT) + .build()) + .dispatch(); + }); } else { commandSender.sendErrorResponse(unsubscribe.getRequestId(), ServerError.MetadataError, "Consumer not found"); @@ -2214,6 +2280,12 @@ protected void handleSeek(CommandSeek seek) { log.info("[{}] [{}][{}] Reset subscription to message id {}", remoteAddress, subscription.getTopic().getName(), subscription.getName(), position); commandSender.sendSuccessResponse(requestId); + newTopicEvent(subscription.getTopicName(), TopicEvent.SUBSCRIPTION_SEEK) + .data(SubscriptionSeekEventData.builder() + .subscriptionName(subscription.getName()) + .messageId(position.toString()) + .build()) + .dispatch(); }).exceptionally(ex -> { log.warn("[{}][{}] Failed to reset subscription: {}", remoteAddress, subscription, ex.getMessage(), ex); @@ -2230,6 +2302,12 @@ protected void handleSeek(CommandSeek seek) { log.info("[{}] [{}][{}] Reset subscription to publish time {}", remoteAddress, subscription.getTopic().getName(), subscription.getName(), timestamp); commandSender.sendSuccessResponse(requestId); + newTopicEvent(subscription.getTopicName(), TopicEvent.SUBSCRIPTION_SEEK) + .data(SubscriptionSeekEventData.builder() + .subscriptionName(subscription.getName()) + .timestamp(timestamp) + .build()) + .dispatch(); }).exceptionally(ex -> { log.warn("[{}][{}] Failed to reset subscription: {}", remoteAddress, subscription, ex.getMessage(), ex); @@ -2301,12 +2379,20 @@ protected void handleCloseProducer(CommandCloseProducer closeProducer) { remoteAddress, producerId); commandSender.sendSuccessResponse(requestId); producers.remove(producerId, producerFuture); + dispatchCloseProducerEvent(producer, DisconnectInitiator.CLIENT); if (brokerInterceptor != null) { brokerInterceptor.producerClosed(this, producer, producer.getMetadata()); } }); } + public TopicEventsDispatcher.TopicEventBuilder newTopicEvent(String topic, TopicEvent topicEvent) { + return getBrokerService().getTopicEventsDispatcher().newEvent(topic, topicEvent) + .role(authRole, originalPrincipal) + .clientVersion(clientVersion) + .proxyVersion(proxyVersion); + } + @Override protected void handleCloseConsumer(CommandCloseConsumer closeConsumer) { checkArgument(state == State.Connected); @@ -2347,6 +2433,7 @@ protected void handleCloseConsumer(CommandCloseConsumer closeConsumer) { consumers.remove(consumerId, consumerFuture); commandSender.sendSuccessResponse(requestId); log.info("[{}] Closed consumer, consumerId={}", remoteAddress, consumerId); + dispatchCloseConsumerEvent(consumer, DisconnectInitiator.CLIENT); if (brokerInterceptor != null) { brokerInterceptor.consumerClosed(this, consumer, consumer.getMetadata()); } @@ -3380,16 +3467,20 @@ protected void interceptCommand(BaseCommand command) throws InterceptException { @Override public void closeProducer(Producer producer) { - // removes producer-connection from map and send close command to producer - safelyRemoveProducer(producer); - closeProducer(producer.getProducerId(), producer.getEpoch(), Optional.empty()); + closeProducer(producer, Optional.empty()); } @Override public void closeProducer(Producer producer, Optional assignedBrokerLookupData) { // removes producer-connection from map and send close command to producer safelyRemoveProducer(producer); - closeProducer(producer.getProducerId(), producer.getEpoch(), assignedBrokerLookupData); + sendCloseProducer(producer, producer.getProducerId(), producer.getEpoch(), assignedBrokerLookupData); + } + + private void sendCloseProducer(Producer producer, long producerId, long epoch, + Optional assignedBrokerLookupData) { + dispatchCloseProducerEvent(producer, DisconnectInitiator.BROKER); + sendCloseProducer(producerId, epoch, assignedBrokerLookupData); } private LookupData getLookupData(BrokerLookupData lookupData) { @@ -3405,7 +3496,7 @@ private LookupData getLookupData(BrokerLookupData lookupData) { } } - private void closeProducer(long producerId, long epoch, Optional assignedBrokerLookupData) { + private void sendCloseProducer(long producerId, long epoch, Optional assignedBrokerLookupData) { if (getRemoteEndpointProtocolVersion() >= v5.getValue()) { assignedBrokerLookupData.ifPresentOrElse(lookup -> { LookupData lookupData = getLookupData(lookup); @@ -3434,10 +3525,20 @@ private void closeProducer(long producerId, long epoch, Optional assignedBrokerLookupData) { // removes consumer-connection from map and send close command to consumer safelyRemoveConsumer(consumer); - closeConsumer(consumer.consumerId(), assignedBrokerLookupData); + closeConsumer(consumer.consumerId(), consumer.consumerName(), consumer.getClientAddress(), + consumer.getSubscription().getTopicName(), consumer.getSubscription().getName(), consumer.subType(), + assignedBrokerLookupData); + } + + private void closeConsumer(long consumerId, String consumerName, String consumerAddress, String topicName, + String subscriptionName, SubType subscriptionType, + Optional assignedBrokerLookupData) { + dispatchCloseConsumerEvent(topicName, consumerId, consumerName, consumerAddress, subscriptionName, + subscriptionType, DisconnectInitiator.BROKER); + sendCloseConsumer(consumerId, assignedBrokerLookupData); } - private void closeConsumer(long consumerId, Optional assignedBrokerLookupData) { + private void sendCloseConsumer(long consumerId, Optional assignedBrokerLookupData) { if (getRemoteEndpointProtocolVersion() >= v5.getValue()) { assignedBrokerLookupData.ifPresentOrElse(lookup -> { LookupData lookupData = getLookupData(lookup); @@ -3450,6 +3551,43 @@ private void closeConsumer(long consumerId, Optional assignedB close(); } } + private void dispatchCloseConsumerEvent(Consumer consumer, DisconnectInitiator initiator) { + dispatchCloseConsumerEvent(consumer.getSubscription().getTopicName(), consumer.consumerId(), + consumer.consumerName(), consumer.getClientAddress(), consumer.getSubscription().getName(), + consumer.subType(), initiator); + } + + private void dispatchCloseConsumerEvent(String topicName, long consumerId, String consumerName, + String consumerAddress, String subscriptionName, + SubType subscriptionType, DisconnectInitiator initiator) { + newTopicEvent(topicName, TopicEvent.CONSUMER_DISCONNECT) + .data(ConsumerDisconnectEventData.builder() + .id(consumerId) + .consumerName(consumerName) + .address(consumerAddress) + .subscriptionName(subscriptionName) + .subscriptionType(subscriptionType) + .initiator(initiator) + .build()) + .dispatch(); + } + + private void dispatchCloseProducerEvent(Producer producer, DisconnectInitiator initiator) { + dispatchCloseProducerEvent(producer.getTopic().getName(), producer.getProducerId(), producer.getProducerName(), + producer.getClientAddress(), initiator); + } + + private void dispatchCloseProducerEvent(String topicName, long producerId, String producerName, + String producerAddress, DisconnectInitiator initiator) { + newTopicEvent(topicName, TopicEvent.PRODUCER_DISCONNECT) + .data(ProducerDisconnectEventData.builder() + .producerId(producerId) + .producerName(producerName) + .address(producerAddress) + .initiator(initiator) + .build()) + .dispatch(); + } /** * It closes the connection with client which triggers {@code channelInactive()} which clears all producers and @@ -3457,7 +3595,18 @@ private void closeConsumer(long consumerId, Optional assignedB */ protected void close() { if (ctx != null) { + ctx.channel().attr(CLOSED_BY_LOCAL_ATTR_KEY).set(true); ctx.close(); + consumers.forEach((n, f) -> { + if (f.isDone()) { + f.thenAccept(c -> dispatchCloseConsumerEvent(c, DisconnectInitiator.BROKER)); + } + }); + producers.forEach((n, f) -> { + if (f.isDone()) { + f.thenAccept(p -> dispatchCloseProducerEvent(p, DisconnectInitiator.BROKER)); + } + }); } } @@ -3835,7 +3984,7 @@ public CompletableFuture> checkConnectionLiveness() { * {@link #channelInactive(ChannelHandlerContext)} event occurs, so skip set it here. */ log.warn("[{}] Connection check timed out. Closing connection.", this.toString()); - ctx.close(); + close(); } else { log.error("[{}] Reached unexpected code block. Completing connection check.", this.toString()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Subscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Subscription.java index 452c30b45febb..41b81ca1412c4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Subscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Subscription.java @@ -76,14 +76,38 @@ default long getNumberOfEntriesDelayed() { CompletableFuture doUnsubscribe(Consumer consumer, boolean forcefully); + /** + * @deprecated Use {@link #purgeBacklog()} instead. + */ + @Deprecated CompletableFuture clearBacklog(); + default CompletableFuture purgeBacklog() { + return clearBacklog().thenApply(v -> 0L); + } + CompletableFuture skipMessages(int numMessagesToSkip); + /** + * @deprecated Use {@link #resetCursorTo(long)} instead. + */ + @Deprecated CompletableFuture resetCursor(long timestamp); + default CompletableFuture resetCursorTo(long timestamp) { + return resetCursor(timestamp).thenApply(v -> null); + } + + /** + * @deprecated Use {@link #resetCursorTo(Position)} instead. + */ + @Deprecated CompletableFuture resetCursor(Position position); + default CompletableFuture resetCursorTo(Position position) { + return resetCursor(position).thenApply(v -> null); + } + CompletableFuture peekNthMessage(int messagePosition); void redeliverUnacknowledgedMessages(Consumer consumer, long consumerEpoch); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index c3d88b9c7237d..61557ba9747e9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -45,8 +45,10 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.event.data.TopicPoliciesUpdateEventData; import org.apache.pulsar.broker.namespace.NamespaceBundleOwnershipListener; import org.apache.pulsar.broker.namespace.NamespaceService; +import org.apache.pulsar.broker.service.TopicEventsListener.TopicEvent; import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.client.api.Message; @@ -434,6 +436,11 @@ private CompletableFuture sendTopicPolicyEventInternal(TopicName topi }); return future; } else { + pulsarService.getBrokerService().getTopicEventsDispatcher() + .newEvent(topicName.getPartitionedTopicName(), TopicEvent.POLICIES_UPDATE) + .data(TopicPoliciesUpdateEventData.builder() + .policiesClass(TopicPolicies.class.getName()).policies(policies).build()) + .dispatch(); return writer.writeAsync(eventKey, event); } }).exceptionally(t -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicEventsDispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicEventsDispatcher.java index a706e00db90bf..2353d7935ac6e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicEventsDispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicEventsDispatcher.java @@ -24,6 +24,12 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CopyOnWriteArrayList; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.service.TopicEventsListener.EventContext; +import org.apache.pulsar.broker.service.TopicEventsListener.EventData; +import org.apache.pulsar.broker.service.TopicEventsListener.EventStage; +import org.apache.pulsar.broker.service.TopicEventsListener.TopicEvent; +import org.apache.pulsar.common.naming.TopicName; /** * Utility class to dispatch topic events. @@ -31,6 +37,12 @@ @Slf4j public class TopicEventsDispatcher { private final List topicEventListeners = new CopyOnWriteArrayList<>(); + private final PulsarService pulsar; + private volatile String brokerId; + + public TopicEventsDispatcher(PulsarService pulsar) { + this.pulsar = pulsar; + } /** * Adds listeners, ignores null listeners. @@ -59,11 +71,13 @@ public void removeTopicEventListener(TopicEventsListener... listeners) { * @param topic * @param event * @param stage + * @deprecated Use {@link #newEvent(String, TopicEvent)} to create an event and then call. */ + @Deprecated public void notify(String topic, TopicEventsListener.TopicEvent event, TopicEventsListener.EventStage stage) { - notify(topic, event, stage, null); + newEvent(topic, event).stage(stage).dispatch(); } /** @@ -72,13 +86,17 @@ public void notify(String topic, * @param event * @param stage * @param t + * @deprecated Use {@link #newEvent(String, TopicEvent)} to create an event and then call. */ + @Deprecated public void notify(String topic, TopicEventsListener.TopicEvent event, TopicEventsListener.EventStage stage, Throwable t) { topicEventListeners - .forEach(listener -> notify(listener, topic, event, stage, t)); + .forEach(listener -> { + newEvent(topic, event).stage(stage).error(t).dispatch(); + }); } /** @@ -92,10 +110,10 @@ public void notify(String topic, public CompletableFuture notifyOnCompletion(CompletableFuture future, String topic, TopicEventsListener.TopicEvent event) { - return future.whenComplete((r, ex) -> notify(topic, - event, - ex == null ? TopicEventsListener.EventStage.SUCCESS : TopicEventsListener.EventStage.FAILURE, - ex)); + return future.whenComplete((r, ex) -> newEvent(topic, event) + .stage(ex == null ? TopicEventsListener.EventStage.SUCCESS : TopicEventsListener.EventStage.FAILURE) + .error(ex) + .dispatch()); } /** @@ -105,33 +123,137 @@ public CompletableFuture notifyOnCompletion(CompletableFuture future, * @param event * @param stage * @param t + * @deprecated Use {@link #newEvent(String, TopicEvent)} to create an event and then call. */ + @Deprecated public static void notify(TopicEventsListener[] listeners, String topic, TopicEventsListener.TopicEvent event, TopicEventsListener.EventStage stage, Throwable t) { Objects.requireNonNull(listeners); - for (TopicEventsListener listener: listeners) { - notify(listener, topic, event, stage, t); + for (TopicEventsListener listener : listeners) { + notify(listener, EventContext.builder() + .topic(topic) + .event(event) + .stage(stage) + .error(t) + .build()); } } private static void notify(TopicEventsListener listener, - String topic, - TopicEventsListener.TopicEvent event, - TopicEventsListener.EventStage stage, - Throwable t) { + TopicEventsListener.EventContext context) { if (listener == null) { return; } try { - listener.handleEvent(topic, event, stage, t); + listener.handleEvent(context); } catch (Throwable ex) { - log.error("TopicEventsListener {} exception while handling {}_{} for topic {}", - listener, event, stage, topic, ex); + log.error("TopicEventsListener {} exception while handling {} for topic {}", + listener, context.getEvent(), context.getTopic(), ex); + } + } + + public TopicEventBuilder newEvent(String topic, TopicEvent event) { + initBrokerId(); + return new TopicEventBuilder(topic, event); + } + + public void notify(TopicEventsListener[] listeners, TopicEventBuilder builder) { + Objects.requireNonNull(listeners); + for (TopicEventsListener listener : listeners) { + builder.dispatch(listener); + } + } + + public class TopicEventBuilder { + private final EventContext.EventContextBuilder builder; + + private TopicEventBuilder(String topic, TopicEvent event) { + TopicName topicName = TopicName.get(topic); + builder = EventContext.builder() + .topic(topicName.getPartitionedTopicName()) + .partition(topicName.getPartitionIndex()) + .cluster(pulsar.getConfiguration().getClusterName()) + .brokerId(brokerId) + .brokerVersion(pulsar.getBrokerVersion()) + .event(event) + .stage(TopicEventsListener.EventStage.SUCCESS); + } + + public TopicEventBuilder role(String role, String originalRole) { + // If originalRole is not null, it indicates the request is made via a proxy. + // In this case: + // - 'role' represents the role of the proxy entity + // - 'originalRole' represents the role of the original client + if (originalRole != null) { + builder.clientRole(originalRole); + builder.proxyRole(role); + } else { + builder.proxyRole(null); + builder.clientRole(role); + } + return this; + } + + public TopicEventBuilder error(Throwable t) { + builder.error(t); + return this; + } + + public TopicEventBuilder clientVersion(String version) { + builder.clientVersion(version); + return this; + } + + public TopicEventBuilder proxyVersion(String version) { + builder.proxyVersion(version); + return this; + } + + public TopicEventBuilder data(EventData data) { + builder.data(data); + return this; + } + + public TopicEventBuilder stage(EventStage stage) { + builder.stage(stage); + return this; + } + + public void dispatch() { + for (TopicEventsListener listener : topicEventListeners) { + dispatch(listener); + } + } + + public void dispatch(TopicEventsListener[] listeners) { + for (TopicEventsListener listener : listeners) { + dispatch(listener); + } + } + + public void dispatch(TopicEventsListener listener) { + EventContext context = builder.build(); + TopicEventsDispatcher.notify(listener, context); } } + private void initBrokerId() { + if (brokerId == null) { + // Lazy initialization of brokerId to avoid blocking the constructor + // in case of PulsarService not being fully initialized. + synchronized (this) { + if (brokerId == null) { + try { + brokerId = pulsar.getBrokerId(); + } catch (Exception ignored) { + // If we cannot get the broker ID, we will leave it as null. + } + } + } + } + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicEventsListener.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicEventsListener.java index 8068067206c66..d0bbe5acfe6f1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicEventsListener.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicEventsListener.java @@ -18,8 +18,16 @@ */ package org.apache.pulsar.broker.service; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.Value; import org.apache.pulsar.common.classification.InterfaceAudience; import org.apache.pulsar.common.classification.InterfaceStability; +import org.apache.pulsar.common.naming.TopicName; /** * Listener for the Topic events. @@ -38,6 +46,31 @@ enum TopicEvent { LOAD, UNLOAD, DELETE, + + LOOKUP, + + TOPIC_METADATA_UPDATE, + + MESSAGE_EXPIRE, + LEDGER_PURGE, + LEDGER_ROLL, + + POLICIES_UPDATE, + POLICIES_APPLY, + + PRODUCER_CONNECT, + PRODUCER_DISCONNECT, + + CONSUMER_CONNECT, + CONSUMER_DISCONNECT, + + SUBSCRIPTION_CREATE, + SUBSCRIPTION_DELETE, + SUBSCRIPTION_SEEK, + SUBSCRIPTION_CLEAR_BACKLOG, + + REPLICATOR_START, + REPLICATOR_STOP } /** @@ -50,6 +83,35 @@ enum EventStage { FAILURE } + @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") + interface EventData { + // Marker interface for event data + } + + @Builder + @Value + @NoArgsConstructor(force = true) + @AllArgsConstructor + class EventContext { + String cluster; + String brokerId; + String proxyRole; + String clientRole; + String topic; + int partition; + TopicEvent event; + EventData data; + EventStage stage; + Throwable error; + String clientVersion; + String brokerVersion; + String proxyVersion; + + // ISO-8601 format + @Builder.Default + String timestamp = OffsetDateTime.now(ZoneId.systemDefault()).toString(); + } + /** * Handle topic event. * Choice of the thread / maintenance of the thread pool is up to the event handlers. @@ -57,6 +119,16 @@ enum EventStage { * @param event - TopicEvent * @param stage - EventStage * @param t - exception in case of FAILURE, if present/known + * @deprecated Use {@link #handleEvent(EventContext)} instead. */ - void handleEvent(String topicName, TopicEvent event, EventStage stage, Throwable t); + @Deprecated + default void handleEvent(String topicName, TopicEvent event, EventStage stage, Throwable t) { + // noop + } + + default void handleEvent(EventContext context) { + handleEvent(context.getPartition() >= 0 + ? TopicName.getTopicPartitionNameString(context.getTopic(), context.getPartition()) + : context.getTopic(), context.getEvent(), context.getStage(), context.getError()); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicLoadingContext.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicLoadingContext.java index 9e3ed230cd26b..645cf18d8c733 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicLoadingContext.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicLoadingContext.java @@ -22,13 +22,13 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import lombok.Builder; import lombok.Getter; -import lombok.RequiredArgsConstructor; import lombok.Setter; import org.apache.pulsar.common.naming.TopicName; import org.jspecify.annotations.Nullable; -@RequiredArgsConstructor +@Builder public class TopicLoadingContext { private static final String EXAMPLE_LATENCY_OUTPUTS = "1234 ms (queued: 567)"; @@ -37,14 +37,24 @@ public class TopicLoadingContext { @Getter private final TopicName topicName; @Getter - private final boolean createIfMissing; + @Setter + private boolean createIfMissing; @Getter - private final CompletableFuture> topicFuture; + @Setter + private CompletableFuture> topicFuture; @Getter @Setter - @Nullable private Map properties; + @Nullable + private Map properties; private long polledFromQueueNs = -1L; + @Getter + @Setter + private String clientVersion; + @Getter + @Setter + private String proxyVersion; + public void polledFromQueue() { polledFromQueueNs = System.nanoTime(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java index 6fbb52047bbff..3a0566ee47972 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java @@ -25,6 +25,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.LongAdder; +import lombok.AllArgsConstructor; import org.apache.bookkeeper.mledger.AsyncCallbacks.FindEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback; import org.apache.bookkeeper.mledger.ManagedCursor; @@ -36,7 +37,9 @@ import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.proto.MLDataFormats; +import org.apache.pulsar.broker.event.data.MessageExpireEventData; import org.apache.pulsar.broker.service.MessageExpirer; +import org.apache.pulsar.broker.service.TopicEventsListener.TopicEvent; import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.protocol.Commands; @@ -205,12 +208,22 @@ public long getTotalMessageExpired() { private static final Logger log = LoggerFactory.getLogger(PersistentMessageExpiryMonitor.class); - private final MarkDeleteCallback markDeleteCallback = new MarkDeleteCallback() { + @AllArgsConstructor + class MessageExpiryMarkDeleteCallback implements MarkDeleteCallback { + private Position position; + @Override public void markDeleteComplete(Object ctx) { long numMessagesExpired = (long) ctx - cursor.getNumberOfEntriesInBacklog(false); msgExpired.recordMultipleEvents(numMessagesExpired, 0 /* no value stats */); totalMsgExpired.add(numMessagesExpired); + topic.getBrokerService().getTopicEventsDispatcher() + .newEvent(topicName, TopicEvent.MESSAGE_EXPIRE) + .data(MessageExpireEventData.builder() + .subscriptionName(subName) + .position(position.toString()) + .build()) + .dispatch(); // If the subscription is a Key_Shared subscription, we should to trigger message dispatch. if (subscription != null && subscription.getType() == SubType.Key_Shared) { subscription.getDispatcher().markDeletePositionMoveForward(); @@ -227,7 +240,12 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { expirationCheckInProgress = FALSE; updateRates(); } - }; + } + + @VisibleForTesting + public MarkDeleteCallback getMarkDeleteCallback(Position position) { + return new MessageExpiryMarkDeleteCallback(position); + } @Override public void findEntryComplete(Position position, Object ctx) { @@ -239,7 +257,7 @@ public void findEntryComplete(Position position, Object ctx) { } log.info("[{}][{}] Expiring all messages until position {}", topicName, subName, position); Position prevMarkDeletePos = cursor.getMarkDeletedPosition(); - cursor.asyncMarkDelete(position, cursor.getProperties(), markDeleteCallback, + cursor.asyncMarkDelete(position, cursor.getProperties(), getMarkDeleteCallback(position), cursor.getNumberOfEntriesInBacklog(false)); if (!Objects.equals(cursor.getMarkDeletedPosition(), prevMarkDeletePos) && subscription != null) { subscription.updateLastMarkDeleteAdvancedTimestamp(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index c37bb3b579617..19c4c5fb080de 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -54,6 +54,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.event.data.ReplicatorStartEventData; import org.apache.pulsar.broker.resourcegroup.ResourceGroup; import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter; import org.apache.pulsar.broker.service.AbstractReplicator; @@ -61,6 +62,7 @@ import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.MessageExpirer; import org.apache.pulsar.broker.service.Replicator; +import org.apache.pulsar.broker.service.TopicEventsListener.TopicEvent; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClientException; @@ -179,6 +181,14 @@ protected void setProducerAndTriggerReadEntries(Producer producer) { // Rewind the cursor to be sure to read again all non-acked messages sent while restarting cursor.rewind(); + localTopic.getBrokerService() + .getTopicEventsDispatcher() + .newEvent(localTopicName, TopicEvent.REPLICATOR_START) + .data(ReplicatorStartEventData.builder() + .replicatorId(replicatorId) + .localCluster(localCluster) + .remoteCluster(remoteCluster).build()) + .dispatch(); // read entries readMoreEntries(); } else { @@ -562,12 +572,21 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { brokerService.executor().schedule(this::readMoreEntries, waitTimeMillis, TimeUnit.MILLISECONDS); } + /** + * @deprecated Use purgeBacklog() instead. + */ + @Deprecated public CompletableFuture clearBacklog() { - CompletableFuture future = new CompletableFuture<>(); + return purgeBacklog().thenAccept(ignored -> { + }); + } + + public CompletableFuture purgeBacklog() { + CompletableFuture future = new CompletableFuture<>(); + long numberOfEntriesInBacklog = cursor.getNumberOfEntriesInBacklog(false); if (log.isDebugEnabled()) { - log.debug("[{}] Backlog size before clearing: {}", replicatorId, - cursor.getNumberOfEntriesInBacklog(false)); + log.debug("[{}] Backlog size before clearing: {}", replicatorId, numberOfEntriesInBacklog); } cursor.asyncClearBacklog(new ClearBacklogCallback() { @@ -577,7 +596,7 @@ public void clearBacklogComplete(Object ctx) { log.debug("[{}] Backlog size after clearing: {}", replicatorId, cursor.getNumberOfEntriesInBacklog(false)); } - future.complete(null); + future.complete(numberOfEntriesInBacklog); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index f0103c4d13fc6..c7923b5c9d58f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -131,7 +131,7 @@ public class PersistentSubscription extends AbstractSubscription { private final PendingAckHandle pendingAckHandle; private volatile Map subscriptionProperties; private volatile CompletableFuture fenceFuture; - private volatile CompletableFuture inProgressResetCursorFuture; + private volatile CompletableFuture inProgressResetCursorFuture; private final ServiceConfiguration config; @@ -237,7 +237,7 @@ protected boolean setReplicated(Boolean replicated, boolean isPersistent) { @Override public CompletableFuture addConsumer(Consumer consumer) { - CompletableFuture inProgressResetCursorFuture = this.inProgressResetCursorFuture; + CompletableFuture inProgressResetCursorFuture = this.inProgressResetCursorFuture; if (inProgressResetCursorFuture != null) { return inProgressResetCursorFuture.handle((ignore, ignoreEx) -> null) .thenCompose(ignore -> addConsumerInternal(consumer)); @@ -746,12 +746,12 @@ public void deleteCursorFailed(ManagedLedgerException exception, Object ctx) { } @Override - public CompletableFuture clearBacklog() { - CompletableFuture future = new CompletableFuture<>(); + public CompletableFuture purgeBacklog() { + CompletableFuture future = new CompletableFuture<>(); + long numberOfEntriesInBacklog = cursor.getNumberOfEntriesInBacklog(false); if (log.isDebugEnabled()) { - log.debug("[{}][{}] Backlog size before clearing: {}", topicName, subName, - cursor.getNumberOfEntriesInBacklog(false)); + log.debug("[{}][{}] Backlog size before clearing: {}", topicName, subName, numberOfEntriesInBacklog); } cursor.asyncClearBacklog(new ClearBacklogCallback() { @@ -766,12 +766,12 @@ public void clearBacklogComplete(Object ctx) { if (ex != null) { future.completeExceptionally(ex); } else { - future.complete(null); + future.complete(numberOfEntriesInBacklog); } }); dispatcher.afterAckMessages(null, ctx); } else { - future.complete(null); + future.complete(numberOfEntriesInBacklog); } } @@ -788,6 +788,11 @@ public void clearBacklogFailed(ManagedLedgerException exception, Object ctx) { return future; } + @Override + public CompletableFuture clearBacklog() { + return purgeBacklog().thenApply(v -> null); + } + @Override public CompletableFuture skipMessages(int numMessagesToSkip) { CompletableFuture future = new CompletableFuture<>(); @@ -826,11 +831,16 @@ public void skipEntriesFailed(ManagedLedgerException exception, Object ctx) { @Override public CompletableFuture resetCursor(long timestamp) { + return resetCursorTo(timestamp).thenApply(v -> null); + } + + @Override + public CompletableFuture resetCursorTo(long timestamp) { if (!IS_FENCED_UPDATER.compareAndSet(PersistentSubscription.this, FALSE, TRUE)) { return CompletableFuture.failedFuture(new SubscriptionBusyException("Failed to fence subscription")); } - final CompletableFuture future = new CompletableFuture<>(); + CompletableFuture future = new CompletableFuture<>(); inProgressResetCursorFuture = future; PersistentMessageFinder persistentMessageFinder = new PersistentMessageFinder(topicName, cursor, config.getManagedLedgerCursorResetLedgerCloseTimestampMaxClockSkewMillis()); @@ -864,7 +874,7 @@ public void findEntryComplete(Position position, Object ctx) { } else { finalPosition = position.getNext(); } - CompletableFuture resetCursorFuture = resetCursorInternal(finalPosition, future, true); + CompletableFuture resetCursorFuture = resetCursorInternal(finalPosition, future, true); FutureUtil.completeAfter(future, resetCursorFuture); } @@ -887,11 +897,16 @@ public void findEntryFailed(ManagedLedgerException exception, @Override public CompletableFuture resetCursor(Position finalPosition) { - final CompletableFuture future = new CompletableFuture<>(); - return resetCursorInternal(finalPosition, future, false); + final CompletableFuture future = new CompletableFuture<>(); + return resetCursorInternal(finalPosition, future, false).thenApply(v -> null); } - private CompletableFuture resetCursorInternal(Position finalPosition, CompletableFuture future, + @Override + public CompletableFuture resetCursorTo(Position finalPosition) { + return resetCursorInternal(finalPosition, new CompletableFuture<>(), false); + } + + public CompletableFuture resetCursorInternal(Position finalPosition, CompletableFuture future, boolean alreadyFenced) { if (!alreadyFenced && !IS_FENCED_UPDATER.compareAndSet(PersistentSubscription.this, FALSE, TRUE)) { @@ -959,7 +974,7 @@ public void resetComplete(Object ctx) { } IS_FENCED_UPDATER.set(PersistentSubscription.this, FALSE); inProgressResetCursorFuture = null; - future.complete(null); + future.complete(finalPosition); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 09041571e7bf5..2f7a550a4a5a3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.time.Clock; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; @@ -76,6 +77,7 @@ import org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.mledger.ManagedLedgerEventListener; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerAlreadyClosedException; import org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException; @@ -89,6 +91,7 @@ import org.apache.bookkeeper.mledger.impl.ManagedCursorContainer; import org.apache.bookkeeper.mledger.impl.ManagedCursorContainer.CursorInfo; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.bookkeeper.mledger.util.Futures; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -96,6 +99,9 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory; import org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerFactory; +import org.apache.pulsar.broker.event.data.LedgerPurgeEventData; +import org.apache.pulsar.broker.event.data.LedgerRollEventData; +import org.apache.pulsar.broker.event.data.TopicPoliciesApplyEventData; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateDataConflictResolver; import org.apache.pulsar.broker.namespace.NamespaceService; @@ -132,6 +138,8 @@ import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.SubscriptionOption; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.TopicEventsListener.EventStage; +import org.apache.pulsar.broker.service.TopicEventsListener.TopicEvent; import org.apache.pulsar.broker.service.TopicPoliciesService; import org.apache.pulsar.broker.service.TransportCnx; import org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage; @@ -408,6 +416,35 @@ public PersistentTopic(String topic, ManagedLedger ledger, BrokerService brokerS ? brokerService.getTopicOrderedExecutor().chooseThread(topic) : null; this.ledger = ledger; + this.ledger.addLedgerEventListener(new ManagedLedgerEventListener() { + @Override + public void onLedgerRoll(LedgerRollEvent event) { + brokerService.getTopicEventsDispatcher() + .newEvent(topic, TopicEvent.LEDGER_ROLL) + .data(LedgerRollEventData.builder().reason(event.getReason()).ledgerId(event.getLedgerId()) + .build()) + .dispatch(); + } + + @Override + public void onLedgerDelete(MLDataFormats.ManagedLedgerInfo.LedgerInfo... ledgerInfos) { + if (ledgerInfos == null || ledgerInfos.length == 0) { + return; + } + + List purgedLedgers = Arrays.stream(ledgerInfos) + .map(n -> LedgerPurgeEventData.LedgerInfo.builder() + .ledgerId(n.getLedgerId()).entries(n.getEntries()) + .timestamp(n.getTimestamp()) + .build()) + .toList(); + brokerService.getTopicEventsDispatcher() + .newEvent(topic, TopicEvent.LEDGER_PURGE) + .data(LedgerPurgeEventData.builder().ledgerInfos(purgedLedgers) + .build()) + .dispatch(); + } + }); this.backloggedCursorThresholdEntries = brokerService.pulsar().getConfiguration().getManagedLedgerCursorBackloggedThreshold(); this.messageDeduplication = new MessageDeduplication(brokerService.pulsar(), this, ledger); @@ -3681,12 +3718,22 @@ public CompletableFuture onPoliciesUpdate(@NonNull Policies data) { applyPolicyTasks.add(applyUpdatedNamespacePolicies()); return FutureUtil.waitForAll(applyPolicyTasks) .thenAccept(__ -> log.info("[{}] namespace-level policies updated successfully", topic)) + .whenComplete((__, ex) -> notifyTopicPoliciesApplyEvent(ex)) .exceptionally(ex -> { log.error("[{}] update namespace polices : {} error", this.getName(), data, ex); throw FutureUtil.wrapToCompletionException(ex); }); } + private void notifyTopicPoliciesApplyEvent(Throwable error) { + brokerService.getTopicEventsDispatcher() + .newEvent(topic, TopicEvent.POLICIES_APPLY) + .error(error) + .stage(error != null ? EventStage.FAILURE : EventStage.SUCCESS) + .data(TopicPoliciesApplyEventData.builder().policies(topicPolicies).build()) + .dispatch(); + } + private CompletableFuture applyUpdatedNamespacePolicies() { return FutureUtil.runWithCurrentThread(() -> updateResourceGroupLimiter()); } @@ -4695,6 +4742,7 @@ public void onUpdate(TopicPolicies policies) { // Apply policies for components(not contains the specified policies which only defined in namespace policies). FutureUtil.waitForAll(applyUpdatedTopicPolicies()) .thenAccept(__ -> log.info("[{}] topic-level policies updated successfully", topic)) + .whenComplete((__, ex) -> notifyTopicPoliciesApplyEvent(ex)) .exceptionally(e -> { Throwable t = FutureUtil.unwrapCompletionException(e); log.error("[{}] update topic-level policy error: {}", topic, t.getMessage(), t); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index 65489eaa34b43..1a36d9c26867a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -167,11 +167,12 @@ public AuthenticationParameters authParams() { * @return the web service caller identification */ public String clientAppId() { - return (String) httpRequest.getAttribute(AuthenticationFilter.AuthenticatedRoleAttributeName); + return httpRequest != null + ? (String) httpRequest.getAttribute(AuthenticationFilter.AuthenticatedRoleAttributeName) : null; } public String originalPrincipal() { - return httpRequest.getHeader(ORIGINAL_PRINCIPAL_HEADER); + return httpRequest != null ? httpRequest.getHeader(ORIGINAL_PRINCIPAL_HEADER) : null; } public AuthenticationDataSource clientAuthData() { @@ -1342,4 +1343,8 @@ protected static void resumeAsyncResponseExceptionally(AsyncResponse asyncRespon asyncResponse.resume(new RestException(realCause)); } } + + protected String getClientVersion() { + return httpRequest != null ? httpRequest.getHeader("User-Agent") : null; + } } diff --git a/pulsar-broker/src/test/java/org/apache/bookkeeper/mledger/impl/MangedLedgerInterceptorImpl2Test.java b/pulsar-broker/src/test/java/org/apache/bookkeeper/mledger/impl/MangedLedgerInterceptorImpl2Test.java index 0a7effc9988cf..1f9631f263ed5 100644 --- a/pulsar-broker/src/test/java/org/apache/bookkeeper/mledger/impl/MangedLedgerInterceptorImpl2Test.java +++ b/pulsar-broker/src/test/java/org/apache/bookkeeper/mledger/impl/MangedLedgerInterceptorImpl2Test.java @@ -25,6 +25,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.mledger.ManagedLedgerEventListener.LedgerRollReason; import org.apache.bookkeeper.mledger.intercept.ManagedLedgerInterceptor; import org.apache.bookkeeper.test.MockedBookKeeperTestCase; import org.apache.pulsar.broker.intercept.ManagedLedgerInterceptorImpl; @@ -43,8 +44,8 @@ public class MangedLedgerInterceptorImpl2Test extends MockedBookKeeperTestCase { private static void switchLedgerManually(ManagedLedgerImpl ledger){ LedgerHandle originalLedgerHandle = ledger.currentLedger; - ledger.ledgerClosed(ledger.currentLedger); - ledger.createLedgerAfterClosed(); + ledger.ledgerClosedWithReason(ledger.currentLedger, LedgerRollReason.FULL); + ledger.createLedgerAfterClosed(LedgerRollReason.FULL); Awaitility.await().until(() -> { return ledger.state == ManagedLedgerImpl.State.LedgerOpened && ledger.currentLedger != originalLedgerHandle; }); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/TopicEventsListenerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/TopicEventsListenerTest.java index 41da87dd16566..9aa9dfd104a4f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/TopicEventsListenerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/TopicEventsListenerTest.java @@ -18,27 +18,46 @@ */ package org.apache.pulsar.broker; +import static org.apache.pulsar.broker.service.TopicEventsListener.TopicEvent.LEDGER_PURGE; +import static org.apache.pulsar.broker.service.TopicEventsListener.TopicEvent.MESSAGE_EXPIRE; import static org.assertj.core.api.Assertions.assertThat; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Optional; import java.util.Queue; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.ArrayUtils; +import org.apache.pulsar.broker.event.data.ProducerDisconnectEventData; import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.TopicEventsListener; +import org.apache.pulsar.broker.service.TopicEventsListener.EventContext; +import org.apache.pulsar.broker.service.TopicEventsListener.EventStage; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.SubscriptionMode; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode; import org.apache.pulsar.common.policies.data.InactiveTopicPolicies; import org.apache.pulsar.common.policies.data.RetentionPolicies; +import org.apache.pulsar.common.util.ObjectMapperFactory; import org.awaitility.Awaitility; -import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; @@ -49,7 +68,7 @@ @Slf4j public class TopicEventsListenerTest extends BrokerTestBase { - final Queue events = new ConcurrentLinkedQueue<>(); + private final Queue events = new ConcurrentLinkedQueue<>(); volatile String topicNameToWatch; String namespace; @@ -83,14 +102,17 @@ protected void setup() throws Exception { super.baseSetup(); pulsar.getConfiguration().setForceDeleteNamespaceAllowed(true); - pulsar.getBrokerService().addTopicEventListener((topic, event, stage, t) -> { - log.info("got event {}__{} for topic {}", event, stage, topic); - if (topic.equals(topicNameToWatch)) { - if (log.isDebugEnabled()) { - log.debug("got event {}__{} for topic {} with detailed stack", - event, stage, topic, new Exception("tracing event source")); + pulsar.getBrokerService().addTopicEventListener(new TopicEventsListener() { + @Override + public void handleEvent(String topic, TopicEvent event, EventStage stage, Throwable t) { + log.info("got event {}__{} for topic {}", event, stage, topic); + if (topic.equals(topicNameToWatch)) { + if (log.isDebugEnabled()) { + log.debug("got event {}__{} for topic {} with detailed stack", + event, stage, topic, new Exception("tracing event source")); + } + events.add(event.toString() + "__" + stage.toString()); } - events.add(event.toString() + "__" + stage.toString()); } }); } @@ -115,6 +137,13 @@ protected void setupTest() throws Exception { events.clear(); } + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + conf.setManagedLedgerMaxEntriesPerLedger(1); + conf.setManagedLedgerMinLedgerRolloverTimeMinutes(1); + } + @AfterMethod(alwaysRun = true) protected void cleanupTest() throws Exception { deleteNamespaceWithRetry(namespace, true); @@ -135,12 +164,10 @@ public void testEvents(String topicTypePersistence, String topicTypePartitioned, } Awaitility.waitAtMost(10, TimeUnit.SECONDS).untilAsserted(() -> - Assert.assertEquals(events.toArray(), new String[]{ - "DELETE__BEFORE", + assertThat(events).containsAll(Arrays.asList("DELETE__BEFORE", "UNLOAD__BEFORE", "UNLOAD__SUCCESS", - "DELETE__SUCCESS" - }) + "DELETE__SUCCESS")) ); } @@ -155,10 +182,8 @@ public void testEventsWithUnload(String topicTypePersistence, String topicTypePa admin.topics().unload(topicName); Awaitility.waitAtMost(10, TimeUnit.SECONDS).untilAsserted(() -> - Assert.assertEquals(events.toArray(), new String[]{ - "UNLOAD__BEFORE", - "UNLOAD__SUCCESS" - }) + assertThat(events.toArray()).containsAll(Arrays.asList("UNLOAD__BEFORE", + "UNLOAD__SUCCESS")) ); events.clear(); @@ -169,14 +194,12 @@ public void testEventsWithUnload(String topicTypePersistence, String topicTypePa } Awaitility.waitAtMost(10, TimeUnit.SECONDS).untilAsserted(() -> - Assert.assertEquals(events.toArray(), new String[]{ - "DELETE__BEFORE", - "DELETE__SUCCESS" - }) + assertThat(events.toArray()).containsAll(Arrays.asList("DELETE__BEFORE", + "DELETE__SUCCESS")) ); } - @Test(dataProvider = "topicType") + @Test(dataProvider = "topicType", groups = "flaky") public void testEventsActiveSub(String topicTypePersistence, String topicTypePartitioned, boolean forceDelete) throws Exception { String topicName = topicTypePersistence + "://" + namespace + "/" + "topic-" + UUID.randomUUID(); @@ -209,6 +232,7 @@ public void testEventsActiveSub(String topicTypePersistence, String topicTypePar if (forceDelete) { expectedEvents = new String[]{ "DELETE__BEFORE", + "PRODUCER_DISCONNECT__SUCCESS", "UNLOAD__BEFORE", "UNLOAD__SUCCESS", "DELETE__SUCCESS", @@ -221,11 +245,7 @@ public void testEventsActiveSub(String topicTypePersistence, String topicTypePar } Awaitility.waitAtMost(10, TimeUnit.SECONDS).untilAsserted(() -> { - // only care about first 4 events max, the rest will be from client recreating deleted topic - String[] eventsToArray = (events.size() <= 4) - ? events.toArray(new String[0]) - : ArrayUtils.subarray(events.toArray(new String[0]), 0, 4); - Assert.assertEquals(eventsToArray, expectedEvents); + assertThat(events.toArray(new String[0])).containsAll(Arrays.stream(expectedEvents).toList()); }); consumer.close(); @@ -252,49 +272,60 @@ public void testTopicAutoGC(String topicTypePersistence, String topicTypePartiti runGC(); - Awaitility.waitAtMost(10, TimeUnit.SECONDS).untilAsserted(() -> - Assert.assertEquals(events.toArray(), new String[]{ + Awaitility.waitAtMost(10, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> + assertThat(events.toArray()).isEqualTo(new String[]{ "UNLOAD__BEFORE", "UNLOAD__SUCCESS", }) ); } + @Test + public void testTopicEventContextSerialization() throws IOException { + ObjectMapper objectMapper = ObjectMapperFactory.getMapper().getObjectMapper(); + EventContext eventContext = EventContext.builder() + .brokerId("broker-1") + .proxyRole("proxy-role") + .clientRole("client-role") + .topic("persistent://prop/namespace/topic") + .stage(EventStage.SUCCESS) + .data(ProducerDisconnectEventData.builder() + .producerId(1) + .address("localhost:1234") + .producerName("abc") + .build()) + .build(); + byte[] bytes = objectMapper.writeValueAsBytes(eventContext); + EventContext deserializedEventContext = objectMapper.readValue(bytes, EventContext.class); + assertEquals(eventContext, deserializedEventContext); + } + private void createTopicAndVerifyEvents(String topicDomain, String topicTypePartitioned, String topicName) throws Exception { final String[] expectedEvents; if (topicDomain.equalsIgnoreCase("persistent") || topicTypePartitioned.equals("partitioned")) { if (topicTypePartitioned.equals("partitioned")) { - if (topicDomain.equalsIgnoreCase("persistent")) { - expectedEvents = new String[]{ - "CREATE__BEFORE", - "CREATE__SUCCESS", - "LOAD__BEFORE", - "LOAD__SUCCESS" - }; - } else { - // For non-persistent partitioned topic, only metadata is initially created; - // partitions are created when the client connects. - // PR #23680 currently records creation events at metadata creation, - // and the broker records them again when partitions are loaded, - // which can result in multiple events. - // Ideally, #23680 should not record the event here, - // because the topic is not fully created until the client connects. - expectedEvents = new String[]{ - "CREATE__BEFORE", - "CREATE__SUCCESS", - "LOAD__BEFORE", - "CREATE__BEFORE", - "CREATE__SUCCESS", - "LOAD__SUCCESS", - }; - } + expectedEvents = new String[]{ + "CREATE__BEFORE", + "CREATE__SUCCESS", + "LOOKUP__SUCCESS", + "LOAD__BEFORE", + "LOAD__SUCCESS", + "PRODUCER_CONNECT__SUCCESS", + "PRODUCER_DISCONNECT__SUCCESS", + "LOOKUP__SUCCESS", + "CONSUMER_CONNECT__SUCCESS", + "CONSUMER_DISCONNECT__SUCCESS", + }; } else { expectedEvents = new String[]{ "LOAD__BEFORE", "CREATE__BEFORE", "CREATE__SUCCESS", - "LOAD__SUCCESS" + "LOAD__SUCCESS", + "LOOKUP__SUCCESS", + "CONSUMER_CONNECT__SUCCESS", + "CONSUMER_DISCONNECT__SUCCESS", }; } } else { @@ -308,7 +339,10 @@ private void createTopicAndVerifyEvents(String topicDomain, String topicTypePart "LOAD__BEFORE", "CREATE__BEFORE", "CREATE__SUCCESS", - "LOAD__SUCCESS" + "LOAD__SUCCESS", + "LOOKUP__SUCCESS", + "CONSUMER_CONNECT__SUCCESS", + "CONSUMER_DISCONNECT__SUCCESS", }; } if (topicTypePartitioned.equals("partitioned")) { @@ -319,9 +353,77 @@ private void createTopicAndVerifyEvents(String topicDomain, String topicTypePart topicNameToWatch = topicName; admin.topics().createNonPartitionedTopic(topicName); } + createConsumer(topicName); Awaitility.waitAtMost(10, TimeUnit.SECONDS).untilAsserted(() -> - Assert.assertEquals(events.toArray(), expectedEvents)); + assertThat(events.toArray()).isEqualTo(expectedEvents)); + } + + @Test(dataProvider = "topicType") + public void testEventsOnSubscription(String topicTypePersistence, String topicTypePartitioned, boolean forceDelete) + throws Exception { + String topicName = topicTypePersistence + "://" + namespace + "/" + "topic-" + UUID.randomUUID(); + + createTopicAndVerifyEvents(topicTypePersistence, topicTypePartitioned, topicName); + + events.clear(); + + if (topicTypePersistence.equals("persistent")) { + admin.topics().createSubscription(topicName, "test-sub", MessageId.earliest); + admin.topics().deleteSubscription(topicName, "test-sub", forceDelete); + + Awaitility.waitAtMost(10, TimeUnit.SECONDS).untilAsserted(() -> + assertThat(events.stream().toList()) + .isEqualTo(Arrays.asList("SUBSCRIPTION_CREATE__SUCCESS", "SUBSCRIPTION_DELETE__SUCCESS")) + ); + } + } + + @Test + public void testTtlAndRetentionEvent() + throws PulsarAdminException, PulsarClientException, ExecutionException, InterruptedException { + String topicName = TopicName.get(namespace + "/testTtlEvent-" + UUID.randomUUID()).toString(); + admin.topics().createNonPartitionedTopic(topicName); + + admin.topicPolicies().setRetention(topicName, new RetentionPolicies(0, 0)); + + String subscriptionName = "testTtlEvent"; + // Create consumer for ttl + @Cleanup + Consumer consumer = + pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscribe(); + consumer.close(); + + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topicName).enableBatching(false).create(); + for (int i = 0; i < 30; i++) { + producer.send(("message-" + i).getBytes(StandardCharsets.UTF_8)); + Thread.sleep(500); + } + + // Unload topic for trim ledgers + admin.topics().unload(topicName); + + int ttl = 3; + Thread.sleep(ttl * 1000); + + this.topicNameToWatch = topicName; + events.clear(); + + CompletableFuture> topicIfExists = pulsar.getBrokerService().getTopicIfExists(topicName); + assertThat(topicIfExists).succeedsWithin(3, TimeUnit.SECONDS); + Optional topic = topicIfExists.get(); + assertThat(topic).isPresent(); + PersistentTopic persistentTopic = (PersistentTopic) topic.get(); + PersistentSubscription subscription = persistentTopic.getSubscription(subscriptionName); + subscription.expireMessages(ttl); + Awaitility.await().untilAsserted(() -> assertThat(events).contains( + MESSAGE_EXPIRE + "__SUCCESS" + )); + pulsar.getBrokerService().checkConsumedLedgers(); + Awaitility.await().untilAsserted(() -> assertThat(events).contains( + LEDGER_PURGE + "__SUCCESS" + )); } @DataProvider(name = "createTopicEventType") @@ -377,6 +479,13 @@ private PulsarAdmin createPulsarAdmin() throws PulsarClientException { .build(); } + private void createConsumer(String topicName) throws PulsarClientException { + Consumer consumer = + pulsarClient.newConsumer().topic(topicName).subscriptionMode(SubscriptionMode.NonDurable) + .subscriptionName("create-consumer").subscribe(); + consumer.close(); + } + private void triggerPartitionsCreation(String topicName) throws Exception { Producer producer = pulsarClient.newProducer() .topic(topicName) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminReplicatorDispatchRateTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminReplicatorDispatchRateTest.java index ce647c2a398b5..a3ababcbaa484 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminReplicatorDispatchRateTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminReplicatorDispatchRateTest.java @@ -174,7 +174,8 @@ public void testReplicatorDispatchRateOnNamespaceAndTopicLevels() throws Excepti // If applied is true, return default dispatch rate, otherwise, return null. admin.topicPolicies().removeReplicatorDispatchRate(topic, r1Cluster); Awaitility.await().untilAsserted(() -> { - assertEquals(admin.topicPolicies().getReplicatorDispatchRate(topic, r1Cluster, true), defaultDispatchRateOnTopic); + assertEquals(admin.topicPolicies().getReplicatorDispatchRate(topic, r1Cluster, true), + defaultDispatchRateOnTopic); assertNull(admin.topicPolicies().getReplicatorDispatchRate(topic, r1Cluster, false)); }); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java index b80605f8f49a3..29cf58f005c0e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java @@ -957,7 +957,8 @@ public void test500Error() throws Exception { future.completeExceptionally(new RuntimeException("500 error contains error message")); NamespaceService namespaceService = pulsar.getNamespaceService(); doReturn(future).when(namespaceService).checkTopicExists(any()); - persistentTopics.createPartitionedTopic(response1, property, cluster, namespace, partitionedTopicName, 5, false); + persistentTopics.createPartitionedTopic(response1, property, cluster, namespace, partitionedTopicName, 5, + false); verify(response1, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ResourceGroupsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ResourceGroupsTest.java index c78c742790dd1..867f8e3cc8ec7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ResourceGroupsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ResourceGroupsTest.java @@ -113,9 +113,9 @@ public void testCrudResourceGroups() throws Exception { }).isInstanceOf(PulsarAdminException.class); // delete the ResourceGroups we created. - Iterator rg_Iterator = expectedRgNames.iterator(); - while (rg_Iterator.hasNext()) { - admin.resourcegroups().deleteResourceGroup(rg_Iterator.next()); + Iterator rgIterator = expectedRgNames.iterator(); + while (rgIterator.hasNext()) { + admin.resourcegroups().deleteResourceGroup(rgIterator.next()); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/RGUsageMTAggrWaitForAllMsgsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/RGUsageMTAggrWaitForAllMsgsTest.java index 6d74ef74528d5..c040429d81e83 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/RGUsageMTAggrWaitForAllMsgsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/RGUsageMTAggrWaitForAllMsgsTest.java @@ -683,13 +683,17 @@ private void verifyRGMetrics(int sentNumBytes, int sentNumMsgs, for (ResourceGroupMonitoringClass mc : ResourceGroupMonitoringClass.values()) { String mcName = mc.name(); int mcIndex = mc.ordinal(); - double quotaBytes = ResourceGroupService.getRgQuotaByteCount(rgName, mcName, pulsar.getConfiguration().getClusterName(),null); + double quotaBytes = ResourceGroupService.getRgQuotaByteCount(rgName, mcName, + pulsar.getConfiguration().getClusterName(), null); totalQuotaBytes[mcIndex] += quotaBytes; - double quotaMesgs = ResourceGroupService.getRgQuotaMessageCount(rgName, mcName,pulsar.getConfiguration().getClusterName(),null); + double quotaMesgs = ResourceGroupService.getRgQuotaMessageCount(rgName, mcName, + pulsar.getConfiguration().getClusterName(), null); totalQuotaMessages[mcIndex] += quotaMesgs; - double usedBytes = ResourceGroupService.getRgLocalUsageByteCount(rgName, mcName,pulsar.getConfiguration().getClusterName(),null); + double usedBytes = ResourceGroupService.getRgLocalUsageByteCount(rgName, mcName, + pulsar.getConfiguration().getClusterName(), null); totalUsedBytes[mcIndex] += usedBytes; - double usedMesgs = ResourceGroupService.getRgLocalUsageMessageCount(rgName, mcName,pulsar.getConfiguration().getClusterName(),null); + double usedMesgs = ResourceGroupService.getRgLocalUsageMessageCount(rgName, mcName, + pulsar.getConfiguration().getClusterName(), null); totalUsedMessages[mcIndex] += usedMesgs; double usageReportedCount = ResourceGroup.getRgUsageReportedCount(rgName, mcName); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupMetricTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupMetricTest.java index bffb17f6d35cf..fe2b85275b5d3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupMetricTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupMetricTest.java @@ -34,15 +34,17 @@ public void testLocalQuotaMetric() { b.bytes = 20; int incTimes = 2; for (int i = 0; i < 2; i++) { - ResourceGroupService.incRgCalculatedQuota(rgName, publish, b, reportPeriod, "local","remote"); + ResourceGroupService.incRgCalculatedQuota(rgName, publish, b, reportPeriod, "local", "remote"); } - double rgLocalUsageByteCount = ResourceGroupService.getRgQuotaByteCount(rgName, publish.name(),"local", "remote"); - double rgQuotaMessageCount = ResourceGroupService.getRgQuotaMessageCount(rgName, publish.name(),"local", "remote"); + double rgLocalUsageByteCount = + ResourceGroupService.getRgQuotaByteCount(rgName, publish.name(), "local", "remote"); + double rgQuotaMessageCount = + ResourceGroupService.getRgQuotaMessageCount(rgName, publish.name(), "local", "remote"); assertEquals(rgLocalUsageByteCount, incTimes * b.bytes * reportPeriod); assertEquals(rgQuotaMessageCount, incTimes * b.messages * reportPeriod); - double rgLocalUsageByte = ResourceGroupService.getRgQuotaByte(rgName, publish.name(),"local", "remote"); - double rgQuotaMessage = ResourceGroupService.getRgQuotaMessage(rgName, publish.name(),"local", "remote"); + double rgLocalUsageByte = ResourceGroupService.getRgQuotaByte(rgName, publish.name(), "local", "remote"); + double rgQuotaMessage = ResourceGroupService.getRgQuotaMessage(rgName, publish.name(), "local", "remote"); assertEquals(rgLocalUsageByte, b.bytes); assertEquals(rgQuotaMessage, b.messages); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManagerTest.java index 3b13f04e66f2b..ca9ee56cb2bf2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManagerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManagerTest.java @@ -54,8 +54,10 @@ public void testReplicationDispatchRateLimiterOnMsgs() { assertTrue(resourceGroupDispatchLimiter.isDispatchRateLimitingEnabled()); - assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnMsg(), resourceGroup.getReplicationDispatchRateInMsgs().longValue()); - assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnMsg(), resourceGroup.getReplicationDispatchRateInMsgs().longValue()); + assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnMsg(), + resourceGroup.getReplicationDispatchRateInMsgs().longValue()); + assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnMsg(), + resourceGroup.getReplicationDispatchRateInMsgs().longValue()); assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnByte(), -1L); assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnByte(), -1L); @@ -73,8 +75,10 @@ public void testReplicationDispatchRateLimiterOnBytes() { assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnMsg(), -1L); assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnMsg(), -1L); - assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnByte(), resourceGroup.getReplicationDispatchRateInBytes().longValue()); - assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnByte(), resourceGroup.getReplicationDispatchRateInBytes().longValue()); + assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnByte(), + resourceGroup.getReplicationDispatchRateInBytes().longValue()); + assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnByte(), + resourceGroup.getReplicationDispatchRateInBytes().longValue()); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java index ea618249e0c39..2ff3ef351aedb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java @@ -63,8 +63,7 @@ protected void setup() throws Exception { @Override public boolean needToReportLocalUsage(long currentBytesUsed, long lastReportedBytes, long currentMessagesUsed, long lastReportedMessages, - long lastReportTimeMSecsSinceEpoch) - { + long lastReportTimeMSecsSinceEpoch) { final int maxSuppressRounds = conf.getResourceUsageMaxUsageReportSuppressRounds(); if (++numLocalReportsEvaluated % maxSuppressRounds == (maxSuppressRounds - 1)) { return true; @@ -103,7 +102,7 @@ public void measureOpsTime() throws PulsarAdminException { long mSecsStart, mSecsEnd, diffMsecs; final int numPerfTestIterations = 1_000; org.apache.pulsar.common.policies.data.ResourceGroup rgConfig = - new org.apache.pulsar.common.policies.data.ResourceGroup(); + new org.apache.pulsar.common.policies.data.ResourceGroup(); BytesAndMessagesCount stats = new BytesAndMessagesCount(); ResourceGroupMonitoringClass monClass; final String rgName = "measureRGIncStatTime"; @@ -137,7 +136,8 @@ public void measureOpsTime() throws PulsarAdminException { for (int ix = 0; ix < numPerfTestIterations; ix++) { for (int monClassIdx = 0; monClassIdx < ResourceGroupMonitoringClass.values().length; monClassIdx++) { monClass = ResourceGroupMonitoringClass.values()[monClassIdx]; - rgs.incrementUsage(tenantName, namespaceName, topicName.toString(), monClass, stats, monClass.equals(ResourceGroupMonitoringClass.ReplicationDispatch) ? "r2" : null); + rgs.incrementUsage(tenantName, namespaceName, topicName.toString(), monClass, stats, + monClass.equals(ResourceGroupMonitoringClass.ReplicationDispatch) ? "r2" : null); } } mSecsEnd = System.currentTimeMillis(); @@ -284,10 +284,10 @@ public void testReplicatorResourceGroupOps() throws PulsarAdminException { Assert.assertNull(r2Limiter.get()); } - @Test + @Test public void testResourceGroupOps() throws PulsarAdminException, InterruptedException { org.apache.pulsar.common.policies.data.ResourceGroup rgConfig = - new org.apache.pulsar.common.policies.data.ResourceGroup(); + new org.apache.pulsar.common.policies.data.ResourceGroup(); final String rgName = "testRG"; final String randomRgName = "Something"; rgConfig.setPublishRateInBytes(15000L); @@ -303,15 +303,15 @@ public void testResourceGroupOps() throws PulsarAdminException, InterruptedExcep Assert.assertThrows(PulsarAdminException.class, () -> rgs.resourceGroupCreate(rgName, rgConfig)); org.apache.pulsar.common.policies.data.ResourceGroup randomConfig = - new org.apache.pulsar.common.policies.data.ResourceGroup(); + new org.apache.pulsar.common.policies.data.ResourceGroup(); Assert.assertThrows(PulsarAdminException.class, () -> rgs.resourceGroupUpdate(randomRgName, randomConfig)); - rgConfig.setPublishRateInBytes(rgConfig.getPublishRateInBytes()*10); - rgConfig.setPublishRateInMsgs(rgConfig.getPublishRateInMsgs()*10); - rgConfig.setDispatchRateInBytes(rgConfig.getDispatchRateInBytes()/10); - rgConfig.setDispatchRateInMsgs(rgConfig.getDispatchRateInMsgs()/10); - rgConfig.setReplicationDispatchRateInBytes(rgConfig.getReplicationDispatchRateInBytes()/10); - rgConfig.setReplicationDispatchRateInMsgs(rgConfig.getReplicationDispatchRateInMsgs()/10); + rgConfig.setPublishRateInBytes(rgConfig.getPublishRateInBytes() * 10); + rgConfig.setPublishRateInMsgs(rgConfig.getPublishRateInMsgs() * 10); + rgConfig.setDispatchRateInBytes(rgConfig.getDispatchRateInBytes() / 10); + rgConfig.setDispatchRateInMsgs(rgConfig.getDispatchRateInMsgs() / 10); + rgConfig.setReplicationDispatchRateInBytes(rgConfig.getReplicationDispatchRateInBytes() / 10); + rgConfig.setReplicationDispatchRateInMsgs(rgConfig.getReplicationDispatchRateInMsgs() / 10); rgs.resourceGroupUpdate(rgName, rgConfig); Assert.assertEquals(rgs.getNumResourceGroups(), 1); @@ -389,7 +389,7 @@ public void testResourceGroupOps() throws PulsarAdminException, InterruptedExcep // calls, or some later round (since the periodic call to calculateQuotaForAllResourceGroups() would be // ongoing). So, we expect bytes/messages setting to be more than 0 and at most numAnonymousQuotaCalculations. Assert.assertTrue(publishQuota.messages > 0 && publishQuota.messages <= numAnonymousQuotaCalculations); - Assert.assertTrue(publishQuota.bytes > 0 && publishQuota.bytes <= numAnonymousQuotaCalculations); + Assert.assertTrue(publishQuota.bytes > 0 && publishQuota.bytes <= numAnonymousQuotaCalculations); // Now it is safe to detach. After this point the service is intentionally idle. rgs.unRegisterTenant(rgName, tenantName); @@ -438,7 +438,8 @@ public void testCleanupStatsWhenUnRegisterTopic() Assert.assertEquals(rgs.getTopicConsumeStats().asMap().size(), 1); Assert.assertEquals(rgs.getReplicationDispatchStats().asMap().size(), 1); Assert.assertEquals(rgs.getTopicToReplicatorsMap().size(), 1); - Set replicators = rgs.getTopicToReplicatorsMap().get(rgs.getTopicToReplicatorsMap().keys().nextElement()); + Set replicators = + rgs.getTopicToReplicatorsMap().get(rgs.getTopicToReplicatorsMap().keys().nextElement()); Assert.assertEquals(replicators.size(), 1); rgs.unRegisterTopic(TopicName.get(topic)); @@ -483,7 +484,8 @@ public void testCleanupStatsWhenUnRegisterNamespace() Assert.assertEquals(rgs.getTopicProduceStats().asMap().size(), 1); Assert.assertEquals(rgs.getTopicConsumeStats().asMap().size(), 1); Assert.assertEquals(rgs.getReplicationDispatchStats().asMap().size(), 1); - Set replicators = rgs.getTopicToReplicatorsMap().get(rgs.getTopicToReplicatorsMap().keys().nextElement()); + Set replicators = + rgs.getTopicToReplicatorsMap().get(rgs.getTopicToReplicatorsMap().keys().nextElement()); Assert.assertEquals(replicators.size(), 1); rgs.unRegisterNameSpace(rgName, topicName.getNamespaceObject()); @@ -502,7 +504,7 @@ public void testCleanupStatsWhenUnRegisterNamespace() * 1) Start periodic tasks by creating a resource group and attaching a namespace. * 2) Assert both futures are non-null (tasks are scheduled) and the schedulersRunning flag is true. * 3) Let try-with-resources close the service, then assert both futures are null, schedulersRunning is false, - * and the resource group map is cleared. + * and the resource group map is cleared. */ @Test(timeOut = 60000) public void testClose() throws Exception { @@ -556,7 +558,7 @@ public void testTopicStatsCache() { BytesAndMessagesCount value = new BytesAndMessagesCount(); cache.put(key, value); Assert.assertEquals(cache.getIfPresent(key), value); - Awaitility.await().pollDelay(ms + 200 , TimeUnit.MILLISECONDS).untilAsserted(() -> { + Awaitility.await().pollDelay(ms + 200, TimeUnit.MILLISECONDS).untilAsserted(() -> { Assert.assertNull(cache.getIfPresent(key)); }); @@ -696,13 +698,16 @@ public void testReplicationDispatchQuotaCalculation() throws PulsarAdminExceptio pulsar, TimeUnit.HOURS, transportManager, new ResourceQuotaCalculatorImpl(pulsar)); String rgName = UUID.randomUUID().toString(); - org.apache.pulsar.common.policies.data.ResourceGroup rgConfig = new org.apache.pulsar.common.policies.data.ResourceGroup(); + org.apache.pulsar.common.policies.data.ResourceGroup rgConfig = + new org.apache.pulsar.common.policies.data.ResourceGroup(); rgConfig.setReplicationDispatchRateInBytes(2000L); rgConfig.setReplicationDispatchRateInMsgs(400L); Map replicatorDispatchRateMap = new HashMap<>(); - replicatorDispatchRateMap.put(getReplicatorDispatchRateLimiterKey("r1"), DispatchRate.builder().dispatchThrottlingRateInByte(1000).dispatchThrottlingRateInMsg(300).build()); - replicatorDispatchRateMap.put(getReplicatorDispatchRateLimiterKey("r3"), DispatchRate.builder().dispatchThrottlingRateInByte(500).dispatchThrottlingRateInMsg(100).build()); + replicatorDispatchRateMap.put(getReplicatorDispatchRateLimiterKey("r1"), + DispatchRate.builder().dispatchThrottlingRateInByte(1000).dispatchThrottlingRateInMsg(300).build()); + replicatorDispatchRateMap.put(getReplicatorDispatchRateLimiterKey("r3"), + DispatchRate.builder().dispatchThrottlingRateInByte(500).dispatchThrottlingRateInMsg(100).build()); rgConfig.setReplicatorDispatchRate(replicatorDispatchRateMap); resourceGroupService.resourceGroupCreate(rgName, rgConfig); @@ -721,12 +726,14 @@ public void testReplicationDispatchQuotaCalculation() throws PulsarAdminExceptio local.addReplicator().setRemoteCluster("r2").setNetworkUsage().setBytesPerPeriod(1400).setMessagesPerPeriod(90); local.addReplicator().setRemoteCluster("r3").setNetworkUsage().setBytesPerPeriod(300).setMessagesPerPeriod(50); local.addReplicator().setRemoteCluster("r4").setNetworkUsage().setBytesPerPeriod(100).setMessagesPerPeriod(10); - local.addReplicator().setRemoteCluster("r5").setNetworkUsage().setBytesPerPeriod(1900).setMessagesPerPeriod(100); + local.addReplicator().setRemoteCluster("r5").setNetworkUsage().setBytesPerPeriod(1900) + .setMessagesPerPeriod(100); // Remote usage ResourceUsage remote = new ResourceUsage(); remote.addReplicator().setRemoteCluster("r1").setNetworkUsage().setBytesPerPeriod(50).setMessagesPerPeriod(20); - remote.addReplicator().setRemoteCluster("r2").setNetworkUsage().setBytesPerPeriod(400).setMessagesPerPeriod(280); + remote.addReplicator().setRemoteCluster("r2").setNetworkUsage().setBytesPerPeriod(400) + .setMessagesPerPeriod(280); remote.addReplicator().setRemoteCluster("r3").setNetworkUsage().setBytesPerPeriod(100).setMessagesPerPeriod(50); remote.addReplicator().setRemoteCluster("r4").setNetworkUsage().setBytesPerPeriod(200).setMessagesPerPeriod(80); remote.addReplicator().setRemoteCluster("r5").setNetworkUsage().setBytesPerPeriod(100).setMessagesPerPeriod(40); @@ -793,9 +800,11 @@ public void testReplicationDispatchQuotaCalculation() throws PulsarAdminExceptio private static final Logger log = LoggerFactory.getLogger(ResourceGroupServiceTest.class); private static final int PUBLISH_INTERVAL_SECS = 500; + private void prepareData() throws PulsarAdminException { admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); } + /** * Helper method to create a fresh ResourceGroupService instance for testing. * Each test should create its own instance to ensure isolation. @@ -811,10 +820,10 @@ private ResourceGroupService createResourceGroupService() { * no periodic tasks are scheduled. * 2) Registering the first attachment (tenant or namespace) starts both periodic tasks. * 3) Updating the publish interval causes rescheduling - * - calling aggregateResourceGroupLocalUsages() reschedules only the aggregation task; - * - calling calculateQuotaForAllResourceGroups() reschedules only the quota-calculation task. + * - calling aggregateResourceGroupLocalUsages() reschedules only the aggregation task; + * - calling calculateQuotaForAllResourceGroups() reschedules only the quota-calculation task. * 4) When the last attachment is unregistered (i.e., no tenants or namespaces remain attached to any RG), - * both periodic tasks are cancelled and their ScheduledFuture fields are cleared. + * both periodic tasks are cancelled and their ScheduledFuture fields are cleared. */ @Test(timeOut = 60000) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationOnTopicLevelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationOnTopicLevelTest.java index 07809c01738ac..3a6c08b8c688f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationOnTopicLevelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationOnTopicLevelTest.java @@ -48,13 +48,14 @@ @Slf4j public class ResourceGroupUsageAggregationOnTopicLevelTest extends ProducerConsumerBase { - private final String TenantName = "pulsar-test"; - private final String NsName = "test"; - private final String TenantAndNsName = TenantName + "/" + NsName; - private final String TestProduceConsumeTopicName = "/test/prod-cons-topic"; - private final String PRODUCE_CONSUME_PERSISTENT_TOPIC = "persistent://" + TenantAndNsName + TestProduceConsumeTopicName; - private final String PRODUCE_CONSUME_NON_PERSISTENT_TOPIC = - "non-persistent://" + TenantAndNsName + TestProduceConsumeTopicName; + private final String tenantName = "pulsar-test"; + private final String nsName = "test"; + private final String tenantAndNsName = tenantName + "/" + nsName; + private final String testProduceConsumeTopicName = "/test/prod-cons-topic"; + private final String produceConsumerPersistentTopic = + "persistent://" + tenantAndNsName + testProduceConsumeTopicName; + private final String produceConsumerNonPersistentTopic = + "non-persistent://" + tenantAndNsName + testProduceConsumeTopicName; @BeforeMethod @Override @@ -64,10 +65,10 @@ protected void setup() throws Exception { final String clusterName = "test"; admin.clusters().createCluster(clusterName, ClusterData.builder().serviceUrl(brokerUrl.toString()).build()); - admin.tenants().createTenant(TenantName, + admin.tenants().createTenant(tenantName, new TenantInfoImpl(Sets.newHashSet("fakeAdminRole"), Sets.newHashSet(clusterName))); - admin.namespaces().createNamespace(TenantAndNsName); - admin.namespaces().setNamespaceReplicationClusters(TenantAndNsName, Sets.newHashSet(clusterName)); + admin.namespaces().createNamespace(tenantAndNsName); + admin.namespaces().setNamespaceReplicationClusters(tenantAndNsName, Sets.newHashSet(clusterName)); } @AfterMethod(alwaysRun = true) @@ -78,12 +79,12 @@ protected void cleanup() throws Exception { @Test public void testPersistentTopicProduceConsumeUsageOnRG() throws Exception { - testProduceConsumeUsageOnRG(PRODUCE_CONSUME_PERSISTENT_TOPIC); + testProduceConsumeUsageOnRG(produceConsumerPersistentTopic); } @Test public void testNonPersistentTopicProduceConsumeUsageOnRG() throws Exception { - testProduceConsumeUsageOnRG(PRODUCE_CONSUME_NON_PERSISTENT_TOPIC); + testProduceConsumeUsageOnRG(produceConsumerNonPersistentTopic); } private void testProduceConsumeUsageOnRG(String topicString) throws Exception { @@ -162,12 +163,12 @@ public void acceptResourceUsage(String broker, ResourceUsage resourceUsage) { TopicName myTopic = TopicName.get(topicString); rgs.unRegisterTopic(myTopic); - rgs.registerTopic(activeRgName,myTopic); + rgs.registerTopic(activeRgName, myTopic); - final int NumMessagesToSend = 10; + final int numMessagesToSend = 10; int sentNumBytes = 0; int sentNumMsgs = 0; - for (int ix = 0; ix < NumMessagesToSend; ix++) { + for (int ix = 0; ix < numMessagesToSend; ix++) { byte[] mesg = String.format("Hi, ix=%s", ix).getBytes(); producer.send(mesg); sentNumBytes += mesg.length; @@ -187,7 +188,7 @@ public void acceptResourceUsage(String broker, ResourceUsage resourceUsage) { recvdNumMsgs++; } - this.verifyStats(rgs,topicString, activeRgName, sentNumBytes, sentNumMsgs, recvdNumBytes, recvdNumMsgs, + this.verifyStats(rgs, topicString, activeRgName, sentNumBytes, sentNumMsgs, recvdNumBytes, recvdNumMsgs, true, true); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java index 56a08eac2096f..e7af46fc44d35 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java @@ -1185,9 +1185,10 @@ public void testTopicLoadingOnDisableNamespaceBundle() throws Exception { pulsar.getNamespaceService().getOwnershipCache().updateBundleState(bundle, false).join(); // try to create topic which should fail as bundle is disable + TopicLoadingContext topicLoadingContext = + TopicLoadingContext.builder().topicName(topic).createIfMissing(true).properties(null).build(); CompletableFuture> futureResult = pulsar.getBrokerService() - .loadOrCreatePersistentTopic(new TopicLoadingContext(topic, true, - new CompletableFuture<>())); + .loadOrCreatePersistentTopic(topicLoadingContext); try { futureResult.get(); @@ -1231,7 +1232,8 @@ public void testConcurrentLoadTopicExceedLimitShouldNotBeAutoCreated() throws Ex for (int i = 0; i < 10; i++) { // try to create topic which should fail as bundle is disable CompletableFuture> futureResult = pulsar.getBrokerService().loadOrCreatePersistentTopic( - new TopicLoadingContext(TopicName.get(topicName + "_" + i), false, new CompletableFuture<>())); + TopicLoadingContext.builder().topicName(TopicName.get(topicName + "_" + i)) + .createIfMissing(false).topicFuture(new CompletableFuture<>()).build()); loadFutures.add(futureResult); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java index 4a7da569ba7f9..cafa551ce242a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java @@ -23,6 +23,8 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; @@ -42,11 +44,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; import javax.ws.rs.client.Entity; import javax.ws.rs.core.MediaType; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedger; @@ -58,7 +60,6 @@ import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo; import org.apache.bookkeeper.test.MockedBookKeeperTestCase; -import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; @@ -78,8 +79,6 @@ import org.apache.pulsar.common.protocol.ByteBufPair; import org.apache.pulsar.common.protocol.Commands; import org.awaitility.Awaitility; -import org.mockito.Mockito; -import org.testng.Assert; import org.testng.annotations.Test; @Test(groups = "broker") @@ -716,26 +715,16 @@ public void testCheckExpiryByLedgerClosureTimeWithAckUnclosedLedger() throws Thr PersistentTopic mock = mock(PersistentTopic.class); when(mock.getName()).thenReturn("topicname"); when(mock.getLastPosition()).thenReturn(PositionFactory.EARLIEST); - PersistentMessageExpiryMonitor monitor = new PersistentMessageExpiryMonitor(mock, c1.getName(), c1, null); - AsyncCallbacks.MarkDeleteCallback markDeleteCallback = - (AsyncCallbacks.MarkDeleteCallback) spy( - FieldUtils.readDeclaredField(monitor, "markDeleteCallback", true)); - FieldUtils.writeField(monitor, "markDeleteCallback", markDeleteCallback, true); - - AtomicReference throwableAtomicReference = new AtomicReference<>(); - Mockito.doAnswer(invocation -> { - ManagedLedgerException argument = invocation.getArgument(0, ManagedLedgerException.class); - throwableAtomicReference.set(argument); - return invocation.callRealMethod(); - }).when(markDeleteCallback).markDeleteFailed(any(), any()); - + PersistentMessageExpiryMonitor monitor = + spy(new PersistentMessageExpiryMonitor(mock, c1.getName(), c1, null)); + MarkDeleteCallback markDeleteCallback = mock(MarkDeleteCallback.class); + doReturn(markDeleteCallback).when(monitor).getMarkDeleteCallback(any()); Position position = ledger.getLastConfirmedEntry(); c1.markDelete(position); Thread.sleep(TimeUnit.SECONDS.toMillis(maxTTLSeconds)); monitor.expireMessages(maxTTLSeconds); assertEquals(c1.getNumberOfEntriesInBacklog(true), 0); - - Assert.assertNull(throwableAtomicReference.get()); + verify(markDeleteCallback, times(0)).markDeleteFailed(any(), any()); } @Test @@ -753,26 +742,17 @@ public void testCheckExpiryAsyncByLedgerClosureTimeWithAckUnclosedLedger() throw } assertEquals(ledger.getLedgersInfoAsList().size(), 2); PersistentTopic mock = mockPersistentTopic("topicname"); - PersistentMessageExpiryMonitor monitor = new PersistentMessageExpiryMonitor(mock, c1.getName(), c1, null); - AsyncCallbacks.MarkDeleteCallback markDeleteCallback = - (AsyncCallbacks.MarkDeleteCallback) spy( - FieldUtils.readDeclaredField(monitor, "markDeleteCallback", true)); - FieldUtils.writeField(monitor, "markDeleteCallback", markDeleteCallback, true); - - AtomicReference throwableAtomicReference = new AtomicReference<>(); - Mockito.doAnswer(invocation -> { - ManagedLedgerException argument = invocation.getArgument(0, ManagedLedgerException.class); - throwableAtomicReference.set(argument); - return invocation.callRealMethod(); - }).when(markDeleteCallback).markDeleteFailed(any(), any()); + PersistentMessageExpiryMonitor monitor = + spy(new PersistentMessageExpiryMonitor(mock, c1.getName(), c1, null)); + MarkDeleteCallback markDeleteCallback = mock(MarkDeleteCallback.class); + doReturn(markDeleteCallback).when(monitor).getMarkDeleteCallback(any()); Position position = ledger.getLastConfirmedEntry(); c1.markDelete(position); Thread.sleep(TimeUnit.SECONDS.toMillis(maxTTLSeconds)); monitor.expireMessagesAsync(maxTTLSeconds).get(); assertEquals(c1.getNumberOfEntriesInBacklog(true), 0); - - Assert.assertNull(throwableAtomicReference.get()); + verify(markDeleteCallback, times(0)).markDeleteFailed(any(), any()); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index 5f124c34658b8..19ccc9d4ec533 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -116,6 +116,7 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.ConnectionPool; +import org.apache.pulsar.client.impl.LookupService; import org.apache.pulsar.client.impl.ProducerBuilderImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; @@ -130,6 +131,7 @@ import org.apache.pulsar.common.api.proto.ProducerAccessMode; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl; @@ -225,14 +227,21 @@ public void setup() throws Exception { doReturn(ctx).when(serverCnx).ctx(); doReturn(CompletableFuture.completedFuture(Optional.of(true))).when(serverCnx).checkConnectionLiveness(); - NamespaceService nsSvc = pulsarTestContext.getPulsarService().getNamespaceService(); + NamespaceService nsSvc = mock(NamespaceService.class); NamespaceBundle bundle = mock(NamespaceBundle.class); doReturn(CompletableFuture.completedFuture(bundle)).when(nsSvc).getBundleAsync(any()); + doReturn(nsSvc).when(pulsarTestContext.getPulsarService()).getNamespaceService(); doReturn(true).when(nsSvc).isServiceUnitOwned(any()); doReturn(CompletableFuture.completedFuture(mock(NamespaceBundle.class))).when(nsSvc).getBundleAsync(any()); doReturn(CompletableFuture.completedFuture(true)).when(nsSvc).checkBundleOwnership(any(), any()); - doReturn(CompletableFuture.completedFuture(TopicExistsInfo.newTopicNotExists())).when(nsSvc) + doReturn(CompletableFuture.completedFuture(TopicExistsInfo.newNonPartitionedTopicExists())).when(nsSvc) .checkTopicExistsAsync(any()); + PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + LookupService lookupService = mock(LookupService.class); + doReturn(CompletableFuture.completedFuture(new PartitionedTopicMetadata(0))).when(lookupService) + .getPartitionedTopicMetadata(any(), anyBoolean(), anyBoolean()); + doReturn(lookupService).when(pulsarClient).getLookup(); + doReturn(pulsarClient).when(pulsarTestContext.getPulsarService()).getClient(); setupMLAsyncCallbackMocks(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRateLimiterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRateLimiterTest.java index 1d6d77e51f65f..46700c418b3b2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRateLimiterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRateLimiterTest.java @@ -82,7 +82,7 @@ enum DispatchRateType { @DataProvider(name = "dispatchRateType") public Object[][] dispatchRateProvider() { - return new Object[][] { { DispatchRateType.messageRate }, { DispatchRateType.byteRate } }; + return new Object[][]{{DispatchRateType.messageRate}, {DispatchRateType.byteRate}}; } @Test @@ -100,7 +100,7 @@ public void testReplicatorRateLimiterWithOnlyTopicLevel() throws Exception { admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2")); @Cleanup PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()) - .statsInterval(0, TimeUnit.SECONDS).build(); + .statsInterval(0, TimeUnit.SECONDS).build(); client1.newProducer().topic(topicName).create().close(); PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getOrCreateTopic(topicName).get(); @@ -110,13 +110,13 @@ public void testReplicatorRateLimiterWithOnlyTopicLevel() throws Exception { //set topic-level policy, which should take effect DispatchRate topicRate = DispatchRate.builder() - .dispatchThrottlingRateInMsg(10) - .dispatchThrottlingRateInByte(20) - .ratePeriodInSecond(30) - .build(); + .dispatchThrottlingRateInMsg(10) + .dispatchThrottlingRateInByte(20) + .ratePeriodInSecond(30) + .build(); admin1.topics().setReplicatorDispatchRate(topicName, topicRate); Awaitility.await().untilAsserted(() -> - assertEquals(admin1.topics().getReplicatorDispatchRate(topicName), topicRate)); + assertEquals(admin1.topics().getReplicatorDispatchRate(topicName), topicRate)); assertTrue(getRateLimiter(topic).isPresent()); assertEquals(getRateLimiter(topic).get().getDispatchRateOnMsg(), 10); assertEquals(getRateLimiter(topic).get().getDispatchRateOnByte(), 20L); @@ -124,10 +124,10 @@ public void testReplicatorRateLimiterWithOnlyTopicLevel() throws Exception { //remove topic-level policy admin1.topics().removeReplicatorDispatchRate(topicName); Awaitility.await().untilAsserted(() -> - assertNull(admin1.topics().getReplicatorDispatchRate(topicName))); + assertNull(admin1.topics().getReplicatorDispatchRate(topicName))); assertEquals(getRateLimiter(topic).get().getDispatchRateOnMsg(), -1); assertEquals(getRateLimiter(topic).get().getDispatchRateOnByte(), - -1L); + -1L); // ResourceGroupDispatchRateLimiter String resourceGroupName = UUID.randomUUID().toString(); @@ -166,7 +166,7 @@ public void testReplicatorRateLimiterWithOnlyNamespaceLevel() throws Exception { admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2")); @Cleanup PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()) - .statsInterval(0, TimeUnit.SECONDS).build(); + .statsInterval(0, TimeUnit.SECONDS).build(); client1.newProducer().topic(topicName).create().close(); PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getOrCreateTopic(topicName).get(); @@ -176,13 +176,13 @@ public void testReplicatorRateLimiterWithOnlyNamespaceLevel() throws Exception { //set namespace-level policy, which should take effect DispatchRate topicRate = DispatchRate.builder() - .dispatchThrottlingRateInMsg(10) - .dispatchThrottlingRateInByte(20) - .ratePeriodInSecond(30) - .build(); + .dispatchThrottlingRateInMsg(10) + .dispatchThrottlingRateInByte(20) + .ratePeriodInSecond(30) + .build(); admin1.namespaces().setReplicatorDispatchRate(namespace, topicRate); Awaitility.await().untilAsserted(() -> - assertEquals(admin1.namespaces().getReplicatorDispatchRate(namespace), topicRate)); + assertEquals(admin1.namespaces().getReplicatorDispatchRate(namespace), topicRate)); assertTrue(getRateLimiter(topic).isPresent()); assertEquals(getRateLimiter(topic).get().getDispatchRateOnMsg(), 10); assertEquals(getRateLimiter(topic).get().getDispatchRateOnByte(), 20L); @@ -190,10 +190,10 @@ public void testReplicatorRateLimiterWithOnlyNamespaceLevel() throws Exception { //remove topic-level policy admin1.namespaces().removeReplicatorDispatchRate(namespace); Awaitility.await().untilAsserted(() -> - assertNull(admin1.namespaces().getReplicatorDispatchRate(namespace))); + assertNull(admin1.namespaces().getReplicatorDispatchRate(namespace))); assertEquals(getRateLimiter(topic).get().getDispatchRateOnMsg(), -1); assertEquals(getRateLimiter(topic).get().getDispatchRateOnByte(), - -1L); + -1L); // ResourceGroupDispatchRateLimiter String resourceGroupName = UUID.randomUUID().toString(); @@ -232,7 +232,7 @@ public void testReplicatorRateLimiterWithOnlyBrokerLevel() throws Exception { admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2")); @Cleanup PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()) - .statsInterval(0, TimeUnit.SECONDS).build(); + .statsInterval(0, TimeUnit.SECONDS).build(); client1.newProducer().topic(topicName).create().close(); PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getOrCreateTopic(topicName).get(); @@ -244,11 +244,11 @@ public void testReplicatorRateLimiterWithOnlyBrokerLevel() throws Exception { admin1.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerReplicatorInByte", "20"); Awaitility.await().untilAsserted(() -> { assertTrue(admin1.brokers() - .getAllDynamicConfigurations().containsKey("dispatchThrottlingRatePerReplicatorInByte")); + .getAllDynamicConfigurations().containsKey("dispatchThrottlingRatePerReplicatorInByte")); assertEquals(admin1.brokers() - .getAllDynamicConfigurations().get("dispatchThrottlingRatePerReplicatorInMsg"), "10"); + .getAllDynamicConfigurations().get("dispatchThrottlingRatePerReplicatorInMsg"), "10"); assertEquals(admin1.brokers() - .getAllDynamicConfigurations().get("dispatchThrottlingRatePerReplicatorInByte"), "20"); + .getAllDynamicConfigurations().get("dispatchThrottlingRatePerReplicatorInByte"), "20"); }); assertTrue(getRateLimiter(topic).isPresent()); @@ -355,12 +355,12 @@ public void testReplicatorRateLimiterDynamicallyChange() throws Exception { @Cleanup PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS) - .build(); + .build(); Producer producer = client1.newProducer().topic(topicName) - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.SinglePartition) - .create(); + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); producer.close(); PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getOrCreateTopic(topicName).get(); @@ -422,7 +422,7 @@ public void testReplicatorRateLimiterDynamicallyChange() throws Exception { * * @throws Exception */ - @Test(dataProvider = "dispatchRateType") + @Test(dataProvider = "dispatchRateType") public void testReplicatorRateLimiterMessageNotReceivedAllMessages(DispatchRateType dispatchRateType) throws Exception { log.info("--- Starting ReplicatorTest::{} --- ", methodName); @@ -454,12 +454,12 @@ public void testReplicatorRateLimiterMessageNotReceivedAllMessages(DispatchRateT @Cleanup PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS) - .build(); + .build(); Producer producer = client1.newProducer().topic(topicName) - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.SinglePartition) - .create(); + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getOrCreateTopic(topicName).get(); @@ -484,16 +484,16 @@ public void testReplicatorRateLimiterMessageNotReceivedAllMessages(DispatchRateT @Cleanup PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()).statsInterval(0, TimeUnit.SECONDS) - .build(); + .build(); final AtomicInteger totalReceived = new AtomicInteger(0); Consumer consumer = client2.newConsumer().topic(topicName) .subscriptionName("sub2-in-cluster2").messageListener((c1, msg) -> { - assertNotNull(msg, "Message cannot be null"); - String receivedMessage = new String(msg.getData()); - log.debug("Received message [{}] in the listener", receivedMessage); - totalReceived.incrementAndGet(); - }).subscribe(); + assertNotNull(msg, "Message cannot be null"); + String receivedMessage = new String(msg.getData()); + log.debug("Received message [{}] in the listener", receivedMessage); + totalReceived.incrementAndGet(); + }).subscribe(); int numMessages = 500; // Asynchronously produce messages @@ -514,7 +514,7 @@ public void testReplicatorRateLimiterMessageNotReceivedAllMessages(DispatchRateT * * 1. verify topic replicator get configured. * 2. namespace setting of replicator dispatchRate, - * verify consumer in other cluster could receive all messages < message limit. + * verify consumer in other cluster could receive all messages < message limit. * 3. verify consumer in other cluster could not receive all messages > message limit. * * @throws Exception @@ -540,12 +540,12 @@ public void testReplicatorRateLimiterMessageReceivedAllMessages() throws Excepti @Cleanup PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS) - .build(); + .build(); Producer producer = client1.newProducer().topic(topicName) - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.SinglePartition) - .create(); + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getOrCreateTopic(topicName).get(); @@ -566,16 +566,16 @@ public void testReplicatorRateLimiterMessageReceivedAllMessages() throws Excepti @Cleanup PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()).statsInterval(0, TimeUnit.SECONDS) - .build(); + .build(); final AtomicInteger totalReceived = new AtomicInteger(0); Consumer consumer = client2.newConsumer().topic(topicName) .subscriptionName("sub2-in-cluster2").messageListener((c1, msg) -> { - assertNotNull(msg, "Message cannot be null"); - String receivedMessage = new String(msg.getData()); - log.debug("Received message [{}] in the listener", receivedMessage); - totalReceived.incrementAndGet(); - }).subscribe(); + assertNotNull(msg, "Message cannot be null"); + String receivedMessage = new String(msg.getData()); + log.debug("Received message [{}] in the listener", receivedMessage); + totalReceived.incrementAndGet(); + }).subscribe(); int numMessages = 50; // Asynchronously produce messages @@ -638,12 +638,13 @@ public void testResourceGroupReplicatorRateLimiter() throws Exception { final AtomicInteger totalReceived = new AtomicInteger(0); @Cleanup - Consumer consumer = client2.newConsumer().topic(topicName).subscriptionName("sub2-in-cluster2").messageListener((c1, msg) -> { - Assert.assertNotNull(msg, "Message cannot be null"); - String receivedMessage = new String(msg.getData()); - log.debug("Received message [{}] in the listener", receivedMessage); - totalReceived.incrementAndGet(); - }).subscribe(); + Consumer consumer = client2.newConsumer().topic(topicName).subscriptionName("sub2-in-cluster2") + .messageListener((c1, msg) -> { + Assert.assertNotNull(msg, "Message cannot be null"); + String receivedMessage = new String(msg.getData()); + log.debug("Received message [{}] in the listener", receivedMessage); + totalReceived.incrementAndGet(); + }).subscribe(); int numMessages = 500; for (int i = 0; i < numMessages; i++) { @@ -657,7 +658,7 @@ public void testResourceGroupReplicatorRateLimiter() throws Exception { public void testLoadResourceGroupReplicatorRateLimiter() throws Exception { final String namespace = "pulsar/replicatormsg-" + System.currentTimeMillis(); final String topicName1 = "persistent://" + namespace + "/" + UUID.randomUUID(); - final String topicName2= "persistent://" + namespace + "/" + UUID.randomUUID(); + final String topicName2 = "persistent://" + namespace + "/" + UUID.randomUUID(); admin1.namespaces().createNamespace(namespace); // 0. set 2 clusters, there will be 1 replicator in each topic @@ -774,7 +775,7 @@ public void testLoadResourceGroupReplicatorRateLimiter() throws Exception { .getResourceGroup(resourceGroupNameOnTopic))); admin1.topicPolicies().setResourceGroup(topicName1, resourceGroupNameOnTopic); admin1.topicPolicies().setResourceGroup(topicName2, resourceGroupNameOnTopic); - Awaitility.await().untilAsserted(() ->{ + Awaitility.await().untilAsserted(() -> { assertEquals(admin1.topicPolicies() .getResourceGroup(topicName1, false), resourceGroupNameOnTopic); assertEquals(admin1.topicPolicies() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index 9044f4a910fb8..f8d796aa17227 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -229,6 +229,8 @@ public void setup() throws Exception { brokerService = pulsarTestContext.getBrokerService(); namespaceService = pulsar.getNamespaceService(); + doReturn(CompletableFuture.completedFuture(TopicExistsInfo.newNonPartitionedTopicExists())) + .when(namespaceService).checkTopicExistsAsync(any()); doReturn(CompletableFuture.completedFuture(mock(NamespaceBundle.class))).when(namespaceService) .getBundleAsync(any()); doReturn(CompletableFuture.completedFuture(true)).when(namespaceService).checkBundleOwnership(any(), any()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TopicTransactionBufferTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TopicTransactionBufferTest.java index 82e5473e28358..476865a56c6d5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TopicTransactionBufferTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TopicTransactionBufferTest.java @@ -179,8 +179,9 @@ public void testCheckDeduplicationFailedWhenCreatePersistentTopic() throws Excep .newTopic(Mockito.eq(topic), Mockito.any(), Mockito.eq(brokerService), Mockito.eq(PersistentTopic.class)); - brokerService.createPersistentTopic0(new TopicLoadingContext(TopicName.get(topic), true, - new CompletableFuture<>())); + brokerService.createPersistentTopic0( + TopicLoadingContext.builder().topicName(TopicName.get(topic)).createIfMissing(true) + .topicFuture(new CompletableFuture<>()).build()); Awaitility.waitAtMost(1, TimeUnit.MINUTES).until(() -> reference.get() != null); PersistentTopic persistentTopic = reference.get(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonDurableSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonDurableSubscriptionTest.java index 5e3a0127e2d91..8787d049374c0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonDurableSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonDurableSubscriptionTest.java @@ -22,8 +22,7 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertTrue; -import java.lang.reflect.Method; +import static org.testng.AssertJUnit.assertTrue; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -32,6 +31,7 @@ import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.mledger.ManagedLedgerEventListener.LedgerRollReason; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; @@ -290,13 +290,6 @@ private void trimLedgers(final String tpName) { } private void switchLedgerManually(final String tpName) throws Exception { - Method ledgerClosed = - ManagedLedgerImpl.class.getDeclaredMethod("ledgerClosed", new Class[]{LedgerHandle.class}); - Method createLedgerAfterClosed = - ManagedLedgerImpl.class.getDeclaredMethod("createLedgerAfterClosed", new Class[0]); - ledgerClosed.setAccessible(true); - createLedgerAfterClosed.setAccessible(true); - // Wait for topic create. org.awaitility.Awaitility.await().untilAsserted(() -> { PersistentTopic persistentTopic = @@ -309,8 +302,8 @@ private void switchLedgerManually(final String tpName) throws Exception { (PersistentTopic) pulsar.getBrokerService().getTopic(tpName, false).join().get(); ManagedLedgerImpl ml = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); LedgerHandle currentLedger1 = WhiteboxImpl.getInternalState(ml, "currentLedger"); - ledgerClosed.invoke(ml, new Object[]{currentLedger1}); - createLedgerAfterClosed.invoke(ml, new Object[0]); + ml.ledgerClosedWithReason(currentLedger1, LedgerRollReason.FULL); + ml.createLedgerAfterClosed(LedgerRollReason.FULL); Awaitility.await().untilAsserted(() -> { LedgerHandle currentLedger2 = WhiteboxImpl.getInternalState(ml, "currentLedger"); assertNotEquals(currentLedger1.getId(), currentLedger2.getId()); @@ -613,7 +606,7 @@ public void testTrimLedgerIfNoDurableCursor() throws Exception { PositionFactory.create(msgIdInDeletedLedger5.getLedgerId(), msgIdInDeletedLedger5.getEntryId()); log.info("Expected mark deleted position: {}", expectedMarkDeletedPos); log.info("Actual mark deleted position: {}", cursorStats.markDeletePosition); - Assert.assertTrue(actMarkDeletedPos.compareTo(expectedMarkDeletedPos) >= 0); + assertTrue(actMarkDeletedPos.compareTo(expectedMarkDeletedPos) >= 0); }); // Clear the incoming queue of the reader for next test. diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceGroups.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceGroups.java index 86c6846499873..2abca603d308e 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceGroups.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceGroups.java @@ -179,7 +179,7 @@ void run() throws PulsarAdminException { } } - @Command(description ="Get replicator rate limiter from a resourcegroup") + @Command(description = "Get replicator rate limiter from a resourcegroup") private class GetReplicatorDispatchRate extends CliCommand { @Parameters(description = "resourcegroup-name", arity = "1") private String resourceGroupName; diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java index 20e94b16effe5..8c7b7e3025e67 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import lombok.Getter; +import lombok.ToString; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType; import org.apache.pulsar.common.policies.data.impl.DispatchRateImpl; @@ -32,6 +33,7 @@ * Topic policy hierarchy value container. */ @Getter +@ToString public class HierarchyTopicPolicies { final PolicyHierarchyValue> replicationClusters; final PolicyHierarchyValue retentionPolicies; diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/PolicyHierarchyValue.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/PolicyHierarchyValue.java index d416f012b7f2f..b4be650da9831 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/PolicyHierarchyValue.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/PolicyHierarchyValue.java @@ -20,11 +20,13 @@ import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import lombok.Getter; +import lombok.ToString; /** * Policy value holder for different hierarchy level. * Currently, we have three hierarchy with priority : topic > namespace > broker. */ +@ToString public class PolicyHierarchyValue { private static final AtomicReferenceFieldUpdater VALUE_UPDATER = AtomicReferenceFieldUpdater.newUpdater(PolicyHierarchyValue.class, Object.class, "value"); From 692a37b3156a73a589b370a726d76f532a0bff0e Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Thu, 30 Oct 2025 16:29:49 +0800 Subject: [PATCH 08/16] [improve][proxy] Refactor broker and proxy auth (#79) # Conflicts: # pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java # pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java # pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java # pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java --- .../authentication/AuthenticationService.java | 13 + .../authentication/BinaryAuthContext.java | 103 ++++ .../authentication/BinaryAuthSession.java | 452 ++++++++++++++++++ .../pulsar/broker/service/ServerCnx.java | 372 +++++--------- .../pulsar/broker/service/ServerCnxTest.java | 234 ++++++--- .../proxy/server/DirectProxyHandler.java | 29 +- .../pulsar/proxy/server/ProxyClientCnx.java | 52 +- .../proxy/server/ProxyConfiguration.java | 6 + .../pulsar/proxy/server/ProxyConnection.java | 271 +++++------ .../proxy/server/ProxyAuthenticationTest.java | 2 +- .../ProxyToProxyAuthenticationTest.java | 167 +++++++ 11 files changed, 1227 insertions(+), 474 deletions(-) create mode 100644 pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/BinaryAuthContext.java create mode 100644 pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/BinaryAuthSession.java create mode 100644 pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyToProxyAuthenticationTest.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationService.java index 5b719bd680145..4ddee9d02ec8a 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationService.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationService.java @@ -252,4 +252,17 @@ public void close() throws IOException { provider.close(); } } + + /** + * Creates a binary authentication session for Pulsar's binary protocol. + *

+ * This method initializes a {@link BinaryAuthSession} using the provided authentication context. + * Both the proxy and broker can use this method to perform binary protocol authentication. + * + * @param ctx the binary authentication context containing authentication state and credentials + * @return a new {@link BinaryAuthSession} instance + */ + public BinaryAuthSession createBinaryAuthSession(BinaryAuthContext ctx) { + return new BinaryAuthSession(ctx); + } } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/BinaryAuthContext.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/BinaryAuthContext.java new file mode 100644 index 0000000000000..5739156764441 --- /dev/null +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/BinaryAuthContext.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authentication; + +import java.net.SocketAddress; +import java.util.concurrent.Executor; +import java.util.function.Supplier; +import javax.net.ssl.SSLSession; +import lombok.Builder; +import lombok.Getter; +import org.apache.pulsar.common.api.proto.CommandConnect; + +/** + * Context object that encapsulates all information required to perform binary protocol + * authentication for a client connection. + *

+ * This context is used by {@link BinaryAuthSession} to manage authentication state, + * credentials, and related connection details during the authentication process. + */ +@Getter +@Builder +public class BinaryAuthContext { + /** + * The {@link CommandConnect} object representing the client's connection request. + */ + private CommandConnect commandConnect; + + /** + * The {@link SSLSession} associated with the connection, if TLS/SSL is used. + * + *

May be {@code null} for non-TLS connections. When present, authenticators + * can use session details (peer certificates, cipher suite, etc.) as part of + * the authentication decision. + */ + private SSLSession sslSession; + + /** + * The {@link AuthenticationService} used to perform authentication operations. + * + *

This is typically the broker-level service that coordinates available + * authentication providers and performs lifecycle operations such as + * verifying credentials or initiating challenges. + */ + private AuthenticationService authenticationService; + + /** + * The executor used to perform asynchronous authentication operations. + *

+ * This should be the Netty event loop executor associated with the current connection, + * ensuring that authentication tasks run on the same event loop thread. + */ + private Executor executor; + + /** + * The remote {@link SocketAddress} of the client initiating the connection. + * + *

This may be used for audit, logging, access control decisions, or for + * binding authentication state to the client's address. + */ + private SocketAddress remoteAddress; + + /** + * Indicates whether to authenticate the client's original authentication data + * instead of simply trusting the provided principal. + *

+ * When set to {@code true}, the session re-validates the original authentication data + * sent by the client. When set to {@code false}, it skips re-authentication + * and only authorizes the provided principal if necessary. + */ + private boolean authenticateOriginalAuthData; + + /** + * A supplier that indicates whether the current connection is still in the + * initial connect phase. + * + *

When this supplier returns {@code true}, the connection is being + * established and the broker should treat the incoming data as part of the + * initial connect handling. When it returns {@code false}, the + * client is already marked connected; subsequent authentication events + * represent refreshes or re-authentication. + * + *

Using a supplier allows deferred evaluation of the initial-connect state + * (for example, if connection state may change between when the context is + * created and when authentication is executed). + */ + private Supplier isInitialConnectSupplier; +} diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/BinaryAuthSession.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/BinaryAuthSession.java new file mode 100644 index 0000000000000..0ccbf46009fef --- /dev/null +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/BinaryAuthSession.java @@ -0,0 +1,452 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authentication; + +import java.util.concurrent.CompletableFuture; +import javax.naming.AuthenticationException; +import lombok.Builder; +import lombok.Getter; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.common.api.AuthData; + +/** + * Represents a per-connection authentication session for a client using the broker's binary protocol. + * + *

This class manages the complete authentication lifecycle for a single client connection. + * It tracks the current {@code AuthenticationState}, authentication provider, method, and role, + * as well as the resolved {@code AuthenticationDataSource}. When a proxy is involved, it can + * also store the original credentials and authentication state forwarded by the proxy. + * + *

{@code BinaryAuthSession} handles both the initial authentication (CONNECT) and + * subsequent re-authentication or credential refresh flows. All asynchronous operations + * are executed using the {@link BinaryAuthContext#getExecutor() executor} provided by the + * associated {@link BinaryAuthContext}. + * + *

The session supports two main connection scenarios: + * + *

Direct client-to-broker connections:

+ *
    + *
  • The client and broker may exchange authentication data multiple times until + * authentication is complete.
    • + *
  • If credentials expire, the broker requests the client to refresh them, + * ensuring that the role remains consistent across refreshes.
    • + *
+ * + *

Client-to-broker connections via a proxy:

+ *
    + *
  • The proxy may optionally forward the original client's authentication data.
    • + *
  • The broker first authenticates the proxy, then optionally validates the + * original client's credentials if forwarded.
    • + *
  • {@code originalAuthState} is non-null when the proxy has forwarded the original + * authentication data and the broker is configured to authenticate it.
    • + *
  • Proxy authentication does not expire. The proxy acts as a transparent intermediary, + * and subsequent client credential refreshes occur directly between the client and broker.
    • + *
+ */ +@Slf4j +@Getter +public class BinaryAuthSession { + private static final byte[] emptyArray = new byte[0]; + + /// Current authentication state of the connected client. + private AuthenticationState authState; + // Authentication method used by the connected client. + private String authMethod; + // Role of the connected client as determined by authentication. + private String authRole = null; + // Authentication data for the connected client (volatile for visibility across threads). + private volatile AuthenticationDataSource authenticationData; + // Authentication provider associated with this session. + private AuthenticationProvider authenticationProvider; + + // Original authentication method forwarded by proxy, if any. + private String originalAuthMethod; + // Original principal forwarded by proxy, if any. + private String originalPrincipal = null; + // Original authentication state forwarded by proxy, if any. + private AuthenticationState originalAuthState; + // Original authentication data forwarded by proxy (volatile for thread visibility). + private volatile AuthenticationDataSource originalAuthData; + // Keep temporarily in order to verify after verifying proxy's authData + private AuthData originalAuthDataCopy; + + // Context holding connection-specific data needed for authentication. + private final BinaryAuthContext context; + + // Default authentication result returned after successful initial authentication + private AuthResult defaultAuthResult; + // Indicates whether the client supports authentication refresh. + private boolean supportsAuthRefresh; + + public BinaryAuthSession(@NonNull BinaryAuthContext context) { + this.context = context; + } + + /** + * Performs the initial authentication process for the client connection. + *

+ * This method handles both standard CONNECT authentication and optional original credentials + * forwarded by a proxy. Authentication may be asynchronous and results in a {@link AuthResult}. + * + * @return a {@link CompletableFuture} that completes with the authentication result + */ + public CompletableFuture doAuthentication() { + var connect = context.getCommandConnect(); + try { + supportsAuthRefresh = connect.getFeatureFlags().hasSupportsAuthRefresh() && connect.getFeatureFlags() + .isSupportsAuthRefresh(); + var authData = connect.hasAuthData() ? connect.getAuthData() : emptyArray; + var clientData = AuthData.of(authData); + // init authentication + if (connect.hasAuthMethodName()) { + authMethod = connect.getAuthMethodName(); + } else if (connect.hasAuthMethod()) { + // Legacy client is passing enum + authMethod = connect.getAuthMethod().name().substring(10).toLowerCase(); + } else { + authMethod = "none"; + } + + defaultAuthResult = AuthResult.builder().clientProtocolVersion(connect.getProtocolVersion()) + .clientVersion(connect.hasClientVersion() ? connect.getClientVersion() : "") + .build(); + + authenticationProvider = context.getAuthenticationService().getAuthenticationProvider(authMethod); + // Not find provider named authMethod. Most used for tests. + // In AuthenticationDisabled, it will set authMethod "none". + if (authenticationProvider == null) { + authRole = context.getAuthenticationService().getAnonymousUserRole() + .orElseThrow(() -> + new AuthenticationException( + "No anonymous role, and no authentication provider configured")); + return CompletableFuture.completedFuture(defaultAuthResult); + } + + authState = + authenticationProvider.newAuthState(clientData, context.getRemoteAddress(), + context.getSslSession()); + + if (log.isDebugEnabled()) { + String role = ""; + if (authState != null && authState.isComplete()) { + role = authState.getAuthRole(); + } else { + role = "authentication incomplete or null"; + } + log.debug("[{}] Authenticate role : {}", context.getRemoteAddress(), role); + } + + if (connect.hasOriginalPrincipal() && context.isAuthenticateOriginalAuthData()) { + // Flow: + // 1. Initialize original authentication. + // 2. Authenticate the proxy's authentication data. + // 3. Authenticate the original authentication data. + if (connect.hasOriginalAuthMethod()) { + originalAuthMethod = connect.getOriginalAuthMethod(); + } else { + originalAuthMethod = "none"; + } + + var originalAuthenticationProvider = + context.getAuthenticationService().getAuthenticationProvider(originalAuthMethod); + + /** + * When both the broker and the proxy are configured with anonymousUserRole + * if the client does not configure an authentication method + * the proxy side will set the value of anonymousUserRole to clientAuthRole when it creates a connection + * and the value of clientAuthMethod will be none. + * Similarly, should also set the value of authRole to anonymousUserRole on the broker side. + */ + if (originalAuthenticationProvider == null) { + authRole = context.getAuthenticationService().getAnonymousUserRole() + .orElseThrow(() -> + new AuthenticationException("No anonymous role, and can't find " + + "AuthenticationProvider for original role using auth method " + + "[" + originalAuthMethod + "] is not available")); + originalPrincipal = authRole; + return CompletableFuture.completedFuture(defaultAuthResult); + } + + originalAuthDataCopy = AuthData.of(connect.getOriginalAuthData().getBytes()); + originalAuthState = originalAuthenticationProvider.newAuthState( + originalAuthDataCopy, + context.getRemoteAddress(), + context.getSslSession()); + } else if (connect.hasOriginalPrincipal()) { + originalPrincipal = connect.getOriginalPrincipal(); + + if (log.isDebugEnabled()) { + log.debug("[{}] Setting original role (forwarded from proxy): {}", + context.getRemoteAddress(), originalPrincipal); + } + } + + return authChallenge(clientData, false, connect.getProtocolVersion(), + connect.hasClientVersion() ? connect.getClientVersion() : ""); + } catch (Exception e) { + return CompletableFuture.failedFuture(e); + } + } + + /** + * Processes the authentication step when the broker receives an authentication response from the client. + * + *

If {@code useOriginalAuthState} is {@code true}, the authentication is performed + * against the original credentials forwarded by a proxy. Otherwise, the primary + * session {@code authState} is used. + */ + public CompletableFuture authChallenge(AuthData clientData, + boolean useOriginalAuthState, + int clientProtocolVersion, + String clientVersion) { + // The original auth state can only be set on subsequent auth attempts (and only + // in presence of a proxy and if the proxy is forwarding the credentials). + // In this case, the re-validation needs to be done against the original client + // credentials. + AuthenticationState authState = useOriginalAuthState ? originalAuthState : this.authState; + String authRole = useOriginalAuthState ? originalPrincipal : this.authRole; + if (log.isDebugEnabled()) { + log.debug("Authenticate using original auth state : {}, role = {}", useOriginalAuthState, authRole); + } + return authState + .authenticateAsync(clientData) + .thenComposeAsync((authChallenge) -> authChallengeSuccessCallback(authChallenge, + useOriginalAuthState, authRole, clientProtocolVersion, clientVersion), + context.getExecutor()); + } + + /** + * Callback invoked when an authentication step completes on the {@link AuthenticationState}. + * + *

If {@code authChallenge} is non-null, the authentication exchange is not yet complete. + * An {@link AuthResult} containing the challenge bytes is returned for the broker or proxy + * to send to the client. This method does not send data itself. + * + *

If {@code authChallenge} is null, the authentication step is complete. In that case, this + * method will: + *

    + *
  • For the initial connection: set the resolved authentication data and role, and + * optionally authenticate original proxy-forwarded credentials.
    • + *
  • For a refresh: validate that the role remains the same and update stored authentication + * data accordingly.
    • + *
+ */ + public CompletableFuture authChallengeSuccessCallback(AuthData authChallenge, + boolean useOriginalAuthState, + String authRole, + int clientProtocolVersion, + String clientVersion) { + try { + if (authChallenge == null) { + // Authentication has completed. It was either: + // 1. the 1st time the authentication process was done, in which case we'll send + // a `CommandConnected` response + // 2. an authentication refresh, in which case we need to refresh authenticationData + AuthenticationState authState = useOriginalAuthState ? originalAuthState : this.authState; + String newAuthRole = authState.getAuthRole(); + AuthenticationDataSource newAuthDataSource = authState.getAuthDataSource(); + + if (context.getIsInitialConnectSupplier().get()) { + // Set the auth data and auth role + if (!useOriginalAuthState) { + this.authRole = newAuthRole; + this.authenticationData = newAuthDataSource; + } + // First time authentication is done + if (originalAuthState != null) { + // We only set originalAuthState when we are going to use it. + return authenticateOriginalData().thenApply( + __ -> defaultAuthResult); + } else { + return CompletableFuture.completedFuture(defaultAuthResult); + } + } else { + // If the connection was already ready, it means we're doing a refresh + if (!StringUtils.isEmpty(authRole)) { + if (!authRole.equals(newAuthRole)) { + log.warn("[{}] Principal cannot change during an authentication refresh expected={} got={}", + context.getRemoteAddress(), authRole, newAuthRole); + return CompletableFuture.failedFuture( + new AuthenticationException("Auth role does not match previous role")); + } + } + // Refresh authentication data + if (!useOriginalAuthState) { + this.authenticationData = newAuthDataSource; + } else { + this.originalAuthData = newAuthDataSource; + } + log.info("[{}] Refreshed authentication credentials for role {}", + context.getRemoteAddress(), authRole); + } + } else { + // auth not complete, continue auth with client side. + return CompletableFuture.completedFuture(AuthResult.builder() + .clientProtocolVersion(clientProtocolVersion) + .clientVersion(clientVersion) + .authMethod(authMethod) + .authData(authChallenge) + .build()); + } + } catch (Exception | AssertionError e) { + return CompletableFuture.failedFuture(e); + } + + return CompletableFuture.completedFuture(defaultAuthResult); + } + + /** + * Performs authentication of the original client credentials forwarded by a proxy. + */ + private CompletableFuture authenticateOriginalData() { + return originalAuthState + .authenticateAsync(originalAuthDataCopy) + .thenComposeAsync((authChallenge) -> { + if (authChallenge != null) { + // The protocol does not yet handle an auth challenge here. + // See https://github.com/apache/pulsar/issues/19291. + return CompletableFuture.failedFuture( + new AuthenticationException("Failed to authenticate original auth data " + + "due to unsupported authChallenge.")); + } else { + try { + // No need to retain these bytes anymore + originalAuthDataCopy = null; + originalAuthData = originalAuthState.getAuthDataSource(); + originalPrincipal = originalAuthState.getAuthRole(); + if (log.isDebugEnabled()) { + log.debug("[{}] Authenticated original role (forwarded from proxy): {}", + context.getRemoteAddress(), originalPrincipal); + } + return CompletableFuture.completedFuture(null); + } catch (Exception | AssertionError e) { + return CompletableFuture.failedFuture(e); + } + } + }, context.getExecutor()); + } + + /** + * Returns whether the current effective authentication state for this session has expired. + * + *

If the session has an {@code originalAuthState} forwarded by a proxy, that state is + * checked first. Otherwise, the session's primary {@code authState} is used. + */ + public boolean isExpired() { + if (originalAuthState != null) { + return originalAuthState.isExpired(); + } + return authState.isExpired(); + } + + /** + * Determines whether the session supports authentication refresh. + * + *

Refresh is not supported when: + *

    + *
  • the client indicated it does not support auth refresh via feature flags
    • + *
  • the session is a proxied connection with an original principal but the proxy did not forward original + * credentials (so re-validation of the original user is impossible)
    • + *
+ */ + public boolean supportsAuthenticationRefresh() { + if (originalPrincipal != null && originalAuthState == null) { + // This case is only checked when the authState is expired because we've reached a point where + // authentication needs to be refreshed, but the protocol does not support it unless the proxy forwards + // the originalAuthData. + log.info( + "[{}] Cannot revalidate user credential when using proxy and" + + " not forwarding the credentials.", + context.getRemoteAddress()); + return false; + } + + if (!supportsAuthRefresh) { + log.warn("[{}] Client doesn't support auth credentials refresh", + context.getRemoteAddress()); + return false; + } + + return true; + } + + /** + * Refreshes the authentication credentials for this session. + * + *

If the session has an {@code originalAuthState} (i.e., credentials forwarded by a proxy + * and broker is configured to authenticate them), the refresh is performed on that state. + * Otherwise, the primary {@code authState} is refreshed. + * + *

The returned {@link AuthResult} contains the updated authentication data and the + * corresponding authentication method. + */ + public AuthResult refreshAuthentication() throws AuthenticationException { + if (originalAuthState != null) { + return AuthResult.builder() + .authData(originalAuthState.refreshAuthentication()) + .authMethod(originalAuthMethod) + .build(); + } + return AuthResult.builder() + .authData(authState.refreshAuthentication()) + .authMethod(authMethod) + .build(); + } + + /** + * Result container for an authentication operation performed by {@link BinaryAuthSession}. + * + *

Holds the optional client protocol/version metadata and the authentication payload + * produced by the underlying authentication provider. This object is returned to the + * broker/proxy to indicate either a completed authentication (no authData) or a pending + * authentication exchange that requires sending {@code authData} back to the client. + */ + @Builder + @Getter + public static class AuthResult { + /** + * Client protocol version used to format protocol-level responses. + * + *

This value is used by the broker or proxy when building response frames so the + * client can interpret any returned authentication bytes correctly. + */ + int clientProtocolVersion; + + /** + * Human-readable client version string, if provided by the client. + */ + String clientVersion; + + /** + * Authentication data produced by the authentication provider. + * + *

When non-null, these bytes represent a challenge or credentials that must be + * sent to the client to continue the authentication handshake. When null, no further + * client exchange is required and authentication is considered complete. + */ + AuthData authData; + + /** + * Identifier of the authentication method associated with {@code authData}. + */ + String authMethod; + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index d96e72df3ed94..2a704c4f8cf75 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -21,7 +21,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static javax.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR; import static javax.ws.rs.core.Response.Status.NOT_FOUND; -import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.pulsar.broker.admin.impl.PersistentTopicsBase.unsafeGetPartitionedTopicMetadataAsync; import static org.apache.pulsar.broker.lookup.TopicLookupBase.lookupTopicAsync; @@ -29,7 +28,6 @@ import static org.apache.pulsar.broker.service.persistent.PersistentTopic.getMigratedClusterUrl; import static org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage.ignoreUnrecoverableBKException; import static org.apache.pulsar.common.api.proto.ProtocolVersion.v5; -import static org.apache.pulsar.common.naming.Constants.WEBSOCKET_DUMMY_ORIGINAL_PRINCIPLE; import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; import static org.apache.pulsar.common.protocol.Commands.newLookupErrorResponse; import com.google.common.annotations.VisibleForTesting; @@ -87,6 +85,9 @@ import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription; import org.apache.pulsar.broker.authentication.AuthenticationProvider; import org.apache.pulsar.broker.authentication.AuthenticationState; +import org.apache.pulsar.broker.authentication.BinaryAuthContext; +import org.apache.pulsar.broker.authentication.BinaryAuthSession; +import org.apache.pulsar.broker.authentication.BinaryAuthSession.AuthResult; import org.apache.pulsar.broker.event.data.ConsumerConnectEventData; import org.apache.pulsar.broker.event.data.ConsumerDisconnectEventData; import org.apache.pulsar.broker.event.data.DisconnectInitiator; @@ -254,9 +255,7 @@ public class ServerCnx extends PulsarHandler implements TransportCnx { private String clientSourceAddressAndPort; private int nonPersistentPendingMessages = 0; private final int maxNonPersistentPendingMessages; - private String originalPrincipal = null; private final boolean schemaValidationEnforced; - private String authMethod = "none"; private final int maxMessageSize; private boolean preciseDispatcherFlowControl; @@ -333,6 +332,7 @@ protected Set initialValue() throws Exception { return Collections.newSetFromMap(new IdentityHashMap<>()); } }; + private BinaryAuthSession binaryAuthSession; enum State { Start, Connected, Failed, Connecting @@ -581,12 +581,12 @@ private CompletableFuture isTopicOperationAllowed(TopicName topicName, return CompletableFuture.completedFuture(true); } CompletableFuture result = service.getAuthorizationService().allowTopicOperationAsync( - topicName, operation, originalPrincipal, authRole, + topicName, operation, getOriginalPrincipal(), getAuthRole(), originalAuthDataSource != null ? originalAuthDataSource : authDataSource, authDataSource); result.thenAccept(isAuthorized -> { if (!isAuthorized) { log.warn("Role {} or OriginalRole {} is not authorized to perform operation {} on topic {}", - authRole, originalPrincipal, operation, topicName); + getAuthRole(), getOriginalPrincipal(), operation, topicName); } }); return result; @@ -596,8 +596,9 @@ private CompletableFuture isTopicOperationAllowed(TopicName topicName, TopicOperation operation) { if (service.isAuthorizationEnabled()) { AuthenticationDataSource authDataSource = - new AuthenticationDataSubscription(authenticationData, subscriptionName); + new AuthenticationDataSubscription(getAuthenticationData(), subscriptionName); AuthenticationDataSource originalAuthDataSource = null; + AuthenticationDataSource originalAuthData = getOriginalAuthData(); if (originalAuthData != null) { originalAuthDataSource = new AuthenticationDataSubscription(originalAuthData, subscriptionName); } @@ -643,6 +644,10 @@ protected void handleLookup(CommandLookupTopic lookupParam) { final Semaphore lookupSemaphore = service.getLookupRequestSemaphore(); if (lookupSemaphore.tryAcquire()) { + String authRole = getAuthRole(); + String originalPrincipal = getOriginalPrincipal(); + AuthenticationDataSource authenticationData = getAuthenticationData(); + AuthenticationDataSource originalAuthData = getOriginalAuthData(); isTopicOperationAllowed(topicName, TopicOperation.LOOKUP, authenticationData, originalAuthData).thenApply( isAuthorized -> { if (isAuthorized) { @@ -732,6 +737,8 @@ protected void handlePartitionMetadataRequest(CommandPartitionedTopicMetadata pa final Semaphore lookupSemaphore = service.getLookupRequestSemaphore(); if (lookupSemaphore.tryAcquire()) { + AuthenticationDataSource authenticationData = getAuthenticationData(); + AuthenticationDataSource originalAuthData = getOriginalAuthData(); isTopicOperationAllowed(topicName, TopicOperation.LOOKUP, authenticationData, originalAuthData).thenApply( isAuthorized -> { if (isAuthorized) { @@ -898,6 +905,9 @@ ByteBuf createConsumerStatsResponse(Consumer consumer, long requestId) { // complete the connect and sent newConnected command private void completeConnect(int clientProtoVersion, String clientVersion) { + String authRole = getAuthRole(); + String originalPrincipal = getOriginalPrincipal(); + String authMethod = getAuthMethod(); if (service.isAuthenticationEnabled()) { if (service.isAuthorizationEnabled()) { if (!service.getAuthorizationService() @@ -947,119 +957,6 @@ private void completeConnect(int clientProtoVersion, String clientVersion) { } } - // According to auth result, send Connected, AuthChallenge, or Error command. - private void doAuthentication(AuthData clientData, - boolean useOriginalAuthState, - int clientProtocolVersion, - final String clientVersion) { - // The original auth state can only be set on subsequent auth attempts (and only - // in presence of a proxy and if the proxy is forwarding the credentials). - // In this case, the re-validation needs to be done against the original client - // credentials. - AuthenticationState authState = useOriginalAuthState ? originalAuthState : this.authState; - String authRole = useOriginalAuthState ? originalPrincipal : this.authRole; - if (log.isDebugEnabled()) { - log.debug("Authenticate using original auth state : {}, role = {}", useOriginalAuthState, authRole); - } - authState - .authenticateAsync(clientData) - .whenCompleteAsync((authChallenge, throwable) -> { - if (throwable == null) { - authChallengeSuccessCallback(authChallenge, useOriginalAuthState, authRole, - clientProtocolVersion, clientVersion); - } else { - authenticationFailed(throwable); - } - }, ctx.executor()); - } - - public void authChallengeSuccessCallback(AuthData authChallenge, - boolean useOriginalAuthState, - String authRole, - int clientProtocolVersion, - String clientVersion) { - try { - if (authChallenge == null) { - // Authentication has completed. It was either: - // 1. the 1st time the authentication process was done, in which case we'll send - // a `CommandConnected` response - // 2. an authentication refresh, in which case we need to refresh authenticationData - AuthenticationState authState = useOriginalAuthState ? originalAuthState : this.authState; - String newAuthRole = authState.getAuthRole(); - AuthenticationDataSource newAuthDataSource = authState.getAuthDataSource(); - - if (state != State.Connected) { - // Set the auth data and auth role - if (!useOriginalAuthState) { - this.authRole = newAuthRole; - this.authenticationData = newAuthDataSource; - } - // First time authentication is done - if (originalAuthState != null) { - // We only set originalAuthState when we are going to use it. - authenticateOriginalData(clientProtocolVersion, clientVersion); - } else { - completeConnect(clientProtocolVersion, clientVersion); - } - } else { - // Refresh the auth data - if (!useOriginalAuthState) { - this.authenticationData = newAuthDataSource; - } else { - this.originalAuthData = newAuthDataSource; - } - // If the connection was already ready, it means we're doing a refresh - if (!StringUtils.isEmpty(authRole)) { - if (!authRole.equals(newAuthRole)) { - log.warn("[{}] Principal cannot change during an authentication refresh expected={} got={}", - remoteAddress, authRole, newAuthRole); - ctx.close(); - } else { - log.info("[{}] Refreshed authentication credentials for role {}", remoteAddress, authRole); - } - } - } - } else { - // auth not complete, continue auth with client side. - ctx.writeAndFlush(Commands.newAuthChallenge(authMethod, authChallenge, clientProtocolVersion)); - if (log.isDebugEnabled()) { - log.debug("[{}] Authentication in progress client by method {}.", remoteAddress, authMethod); - } - } - } catch (Exception | AssertionError e) { - authenticationFailed(e); - } - } - - private void authenticateOriginalData(int clientProtoVersion, String clientVersion) { - originalAuthState - .authenticateAsync(originalAuthDataCopy) - .whenCompleteAsync((authChallenge, throwable) -> { - if (throwable != null) { - authenticationFailed(throwable); - } else if (authChallenge != null) { - // The protocol does not yet handle an auth challenge here. - // See https://github.com/apache/pulsar/issues/19291. - authenticationFailed(new AuthenticationException("Failed to authenticate original auth data " - + "due to unsupported authChallenge.")); - } else { - try { - // No need to retain these bytes anymore - originalAuthDataCopy = null; - originalAuthData = originalAuthState.getAuthDataSource(); - originalPrincipal = originalAuthState.getAuthRole(); - if (log.isDebugEnabled()) { - log.debug("[{}] Authenticated original role (forwarded from proxy): {}", - remoteAddress, originalPrincipal); - } - completeConnect(clientProtoVersion, clientVersion); - } catch (Exception | AssertionError e) { - authenticationFailed(e); - } - } - }, ctx.executor()); - } - // Handle authentication and authentication refresh failures. Must be called from event loop. private void authenticationFailed(Throwable t) { String operation; @@ -1093,26 +990,15 @@ private void maybeScheduleAuthenticationCredentialsRefresh() { private void refreshAuthenticationCredentials() { assert ctx.executor().inEventLoop(); - AuthenticationState authState = this.originalAuthState != null ? originalAuthState : this.authState; if (getState() == State.Failed) { // Happens when an exception is thrown that causes this connection to close. return; - } else if (!authState.isExpired()) { + } else if (!binaryAuthSession.isExpired()) { // Credentials are still valid. Nothing to do at this point return; - } else if (originalPrincipal != null && originalAuthState == null) { - // This case is only checked when the authState is expired because we've reached a point where - // authentication needs to be refreshed, but the protocol does not support it unless the proxy forwards - // the originalAuthData. - log.info( - "[{}] Cannot revalidate user credential when using proxy and" - + " not forwarding the credentials. Closing connection", - remoteAddress); - ctx.close(); - return; } - if (!supportsAuthenticationRefresh()) { + if (!binaryAuthSession.supportsAuthenticationRefresh()) { log.warn("[{}] Closing connection because client doesn't support auth credentials refresh", remoteAddress); ctx.close(); @@ -1127,11 +1013,12 @@ private void refreshAuthenticationCredentials() { } log.info("[{}] Refreshing authentication credentials for originalPrincipal {} and authRole {}", - remoteAddress, originalPrincipal, this.authRole); + remoteAddress, getOriginalPrincipal(), getAuthRole()); try { - AuthData brokerData = authState.refreshAuthentication(); - - writeAndFlush(Commands.newAuthChallenge(authMethod, brokerData, + AuthResult refreshAuthentication = binaryAuthSession.refreshAuthentication(); + String authMethod = refreshAuthentication.getAuthMethod(); + writeAndFlush(Commands.newAuthChallenge(authMethod, + refreshAuthentication.getAuthData(), getRemoteEndpointProtocolVersion())); if (log.isDebugEnabled()) { log.debug("[{}] Sent auth challenge to client to refresh credentials with method: {}.", @@ -1146,8 +1033,6 @@ private void refreshAuthenticationCredentials() { } } - private static final byte[] emptyArray = new byte[0]; - @Override protected void handleConnect(CommandConnect connect) { checkArgument(state == State.Start); @@ -1196,31 +1081,6 @@ protected void handleConnect(CommandConnect connect) { state = State.Connecting; try { - byte[] authData = connect.hasAuthData() ? connect.getAuthData() : emptyArray; - AuthData clientData = AuthData.of(authData); - // init authentication - if (connect.hasAuthMethodName()) { - authMethod = connect.getAuthMethodName(); - } else if (connect.hasAuthMethod()) { - // Legacy client is passing enum - authMethod = connect.getAuthMethod().name().substring(10).toLowerCase(); - } else { - authMethod = "none"; - } - - authenticationProvider = getBrokerService() - .getAuthenticationService() - .getAuthenticationProvider(authMethod); - - // Not find provider named authMethod. Most used for tests. - // In AuthenticationDisabled, it will set authMethod "none". - if (authenticationProvider == null) { - authRole = getBrokerService().getAuthenticationService().getAnonymousUserRole() - .orElseThrow(() -> - new AuthenticationException("No anonymous role, and no authentication provider configured")); - completeConnect(clientProtocolVersion, clientVersion); - return; - } // init authState and other var ChannelHandler sslHandler = ctx.channel().pipeline().get(PulsarChannelInitializer.TLS_HANDLER); SSLSession sslSession = null; @@ -1228,68 +1088,23 @@ protected void handleConnect(CommandConnect connect) { sslSession = ((SslHandler) sslHandler).engine().getSession(); } - authState = authenticationProvider.newAuthState(clientData, remoteAddress, sslSession); - - if (log.isDebugEnabled()) { - String role = ""; - if (authState != null && authState.isComplete()) { - role = authState.getAuthRole(); - } else { - role = "authentication incomplete or null"; - } - log.debug("[{}] Authenticate role : {}", remoteAddress, role); - } - - if (connect.hasOriginalPrincipal() && service.getPulsar().getConfig().isAuthenticateOriginalAuthData() - && !WEBSOCKET_DUMMY_ORIGINAL_PRINCIPLE.equals(connect.getOriginalPrincipal())) { - // Flow: - // 1. Initialize original authentication. - // 2. Authenticate the proxy's authentication data. - // 3. Authenticate the original authentication data. - String originalAuthMethod; - if (connect.hasOriginalAuthMethod()) { - originalAuthMethod = connect.getOriginalAuthMethod(); - } else { - originalAuthMethod = "none"; - } - - AuthenticationProvider originalAuthenticationProvider = getBrokerService() - .getAuthenticationService() - .getAuthenticationProvider(originalAuthMethod); - - /** - * When both the broker and the proxy are configured with anonymousUserRole - * if the client does not configure an authentication method - * the proxy side will set the value of anonymousUserRole to clientAuthRole when it creates a connection - * and the value of clientAuthMethod will be none. - * Similarly, should also set the value of authRole to anonymousUserRole on the broker side. - */ - if (originalAuthenticationProvider == null) { - authRole = getBrokerService().getAuthenticationService().getAnonymousUserRole() - .orElseThrow(() -> - new AuthenticationException("No anonymous role, and can't find " - + "AuthenticationProvider for original role using auth method " - + "[" + originalAuthMethod + "] is not available")); - originalPrincipal = authRole; - completeConnect(clientProtocolVersion, clientVersion); - return; - } - - originalAuthDataCopy = AuthData.of(connect.getOriginalAuthData().getBytes()); - originalAuthState = originalAuthenticationProvider.newAuthState( - originalAuthDataCopy, - remoteAddress, - sslSession); - } else if (connect.hasOriginalPrincipal()) { - originalPrincipal = connect.getOriginalPrincipal(); - - if (log.isDebugEnabled()) { - log.debug("[{}] Setting original role (forwarded from proxy): {}", - remoteAddress, originalPrincipal); - } - } - - doAuthentication(clientData, false, clientProtocolVersion, clientVersion); + binaryAuthSession = service.getAuthenticationService().createBinaryAuthSession(BinaryAuthContext.builder() + .executor(ctx.executor()) + .remoteAddress(remoteAddress) + .sslSession(sslSession) + .authenticationService(service.getAuthenticationService()) + .commandConnect(connect) + .isInitialConnectSupplier(() -> state != State.Connected) + .authenticateOriginalAuthData(service.getPulsar().getConfig().isAuthenticateOriginalAuthData()) + .build()); + binaryAuthSession.doAuthentication() + .whenCompleteAsync((authResult, ex) -> { + if (ex != null) { + authenticationFailed(ex); + } else { + handleAuthResult(authResult); + } + }, ctx.executor()); } catch (Exception e) { authenticationFailed(e); } @@ -1308,14 +1123,44 @@ protected void handleAuthResponse(CommandAuthResponse authResponse) { } try { - AuthData clientData = AuthData.of(authResponse.getResponse().getAuthData()); - doAuthentication(clientData, originalAuthState != null, authResponse.getProtocolVersion(), - authResponse.hasClientVersion() ? authResponse.getClientVersion() : EMPTY); + if (binaryAuthSession != null) { + AuthData clientData = AuthData.of(authResponse.getResponse().getAuthData()); + binaryAuthSession.authChallenge(clientData, binaryAuthSession.getOriginalAuthState() != null, + authResponse.getProtocolVersion(), + authResponse.hasClientVersion() ? authResponse.getClientVersion() : "") + .whenCompleteAsync((authResult, ex) -> { + if (ex != null) { + authenticationFailed(ex); + } else { + handleAuthResult(authResult); + } + }, ctx.executor()); + } else { + authenticationFailed(new AuthenticationException("Authentication session is null or not initialized")); + } } catch (Exception e) { authenticationFailed(e); } } + private void handleAuthResult(AuthResult authResult) { + AuthData authData = authResult.getAuthData(); + if (authData != null) { + writeAndFlush(Commands.newAuthChallenge( + authResult.getAuthMethod(), + authData, + authResult.getClientProtocolVersion())); + if (log.isDebugEnabled()) { + log.debug("[{}] Authentication in progress client by method {}.", remoteAddress, + authResult.getAuthMethod()); + } + } else { + if (state == State.Connecting) { + completeConnect(authResult.getClientProtocolVersion(), authResult.getClientVersion()); + } + } + } + @Override protected void handleSubscribe(final CommandSubscribe subscribe) { checkArgument(state == State.Connected); @@ -1328,7 +1173,7 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { if (log.isDebugEnabled()) { log.debug("[{}] Handle subscribe command: auth role = {}, original auth role = {}", - remoteAddress, authRole, originalPrincipal); + remoteAddress, getAuthRole(), getOriginalPrincipal()); } final String subscriptionName = subscribe.getSubscription(); @@ -1629,7 +1474,7 @@ private SchemaData getSchema(Schema protocolSchema) { .data(protocolSchema.getSchemaData()) .isDeleted(false) .timestamp(System.currentTimeMillis()) - .user(Strings.nullToEmpty(originalPrincipal)) + .user(Strings.nullToEmpty(getOriginalPrincipal())) .type(Commands.getSchemaType(protocolSchema.getType())) .props(protocolSchema.getPropertiesList().stream().collect( Collectors.toMap( @@ -1667,7 +1512,7 @@ protected void handleProducer(final CommandProducer cmdProducer) { } CompletableFuture isAuthorizedFuture = isTopicOperationAllowed( - topicName, TopicOperation.PRODUCE, authenticationData, originalAuthData + topicName, TopicOperation.PRODUCE, getAuthenticationData(), getOriginalAuthData() ); if (!Strings.isNullOrEmpty(initialSubscriptionName)) { @@ -2388,7 +2233,7 @@ protected void handleCloseProducer(CommandCloseProducer closeProducer) { public TopicEventsDispatcher.TopicEventBuilder newTopicEvent(String topic, TopicEvent topicEvent) { return getBrokerService().getTopicEventsDispatcher().newEvent(topic, topicEvent) - .role(authRole, originalPrincipal) + .role(getAuthRole(), getOriginalPrincipal()) .clientVersion(clientVersion) .proxyVersion(proxyVersion); } @@ -2662,6 +2507,10 @@ private CompletableFuture isNamespaceOperationAllowed(NamespaceName nam return CompletableFuture.completedFuture(true); } CompletableFuture isProxyAuthorizedFuture; + String originalPrincipal = getOriginalPrincipal(); + AuthenticationDataSource originalAuthData = getOriginalAuthData(); + String authRole = getAuthRole(); + AuthenticationDataSource authenticationData = getAuthData(); if (originalPrincipal != null) { isProxyAuthorizedFuture = service.getAuthorizationService().allowNamespaceOperationAsync( namespaceName, operation, originalPrincipal, originalAuthData); @@ -3109,11 +2958,13 @@ protected void handleEndTxn(CommandEndTxn command) { private CompletableFuture isSuperUser() { assert ctx.executor().inEventLoop(); if (service.isAuthenticationEnabled() && service.isAuthorizationEnabled()) { + AuthenticationDataSource originalAuthData = getOriginalAuthData(); + AuthenticationDataSource authenticationData = getAuthData(); CompletableFuture isAuthRoleAuthorized = service.getAuthorizationService().isSuperUser( - authRole, authenticationData); - if (originalPrincipal != null) { + getAuthRole(), authenticationData); + if (getOriginalPrincipal() != null) { CompletableFuture isOriginalPrincipalAuthorized = service.getAuthorizationService() - .isSuperUser(originalPrincipal, + .isSuperUser(getOriginalPrincipal(), originalAuthData != null ? originalAuthData : authenticationData); return isOriginalPrincipalAuthorized.thenCombine(isAuthRoleAuthorized, (originalPrincipal, authRole) -> originalPrincipal && authRole); @@ -3854,11 +3705,6 @@ public boolean isBatchMessageCompatibleVersion() { return getRemoteEndpointProtocolVersion() >= ProtocolVersion.v4.getValue(); } - boolean supportsAuthenticationRefresh() { - return features != null && features.isSupportsAuthRefresh(); - } - - boolean supportBrokerMetadata() { return features != null && features.isSupportsBrokerEntryMetadata(); } @@ -3883,29 +3729,44 @@ public boolean isPreciseDispatcherFlowControl() { } public AuthenticationState getAuthState() { - return authState; + return binaryAuthSession != null ? binaryAuthSession.getAuthState() : null; } @Override public AuthenticationDataSource getAuthenticationData() { - return originalAuthData != null ? originalAuthData : authenticationData; + if (binaryAuthSession == null) { + return null; + } + return binaryAuthSession.getAuthenticationData(); } public String getPrincipal() { - return originalPrincipal != null ? originalPrincipal : authRole; + if (binaryAuthSession == null) { + return null; + } + String originalPrincipal = binaryAuthSession.getOriginalPrincipal(); + if (originalPrincipal != null) { + return originalPrincipal; + } + return binaryAuthSession.getAuthRole(); } public AuthenticationProvider getAuthenticationProvider() { - return authenticationProvider; + return binaryAuthSession != null ? binaryAuthSession.getAuthenticationProvider() : null; } @Override public String getAuthRole() { - return authRole; + return binaryAuthSession != null ? binaryAuthSession.getAuthRole() : null; } public String getAuthMethod() { - return authMethod; + return binaryAuthSession != null ? binaryAuthSession.getAuthMethod() : null; + } + + @VisibleForTesting + public BinaryAuthSession getBinaryAuthSession() { + return binaryAuthSession; } public ConcurrentLongHashMap> getConsumers() { @@ -4050,31 +3911,26 @@ public boolean hasProducers() { @VisibleForTesting protected String getOriginalPrincipal() { - return originalPrincipal; + return binaryAuthSession != null ? binaryAuthSession.getOriginalPrincipal() : null; } @VisibleForTesting protected AuthenticationDataSource getAuthData() { - return authenticationData; + return binaryAuthSession != null ? binaryAuthSession.getAuthenticationData() : null; } @VisibleForTesting protected AuthenticationDataSource getOriginalAuthData() { - return originalAuthData; + return binaryAuthSession != null ? binaryAuthSession.getOriginalAuthData() : null; } @VisibleForTesting protected AuthenticationState getOriginalAuthState() { - return originalAuthState; - } - - @VisibleForTesting - protected void setAuthRole(String authRole) { - this.authRole = authRole; + return binaryAuthSession != null ? binaryAuthSession.getOriginalAuthState() : null; } @VisibleForTesting - void setAuthState(AuthenticationState authState) { - this.authState = authState; + void clearBinaryAuthSession() { + this.binaryAuthSession = null; } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index f8d796aa17227..f0c79230d88c3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -35,6 +35,7 @@ import static org.mockito.Mockito.matches; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -102,7 +103,8 @@ import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription; import org.apache.pulsar.broker.authentication.AuthenticationProvider; import org.apache.pulsar.broker.authentication.AuthenticationService; -import org.apache.pulsar.broker.authentication.AuthenticationState; +import org.apache.pulsar.broker.authentication.BinaryAuthContext; +import org.apache.pulsar.broker.authentication.BinaryAuthSession; import org.apache.pulsar.broker.authorization.AuthorizationService; import org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider; import org.apache.pulsar.broker.namespace.NamespaceService; @@ -458,6 +460,9 @@ public void testConnectCommandWithAuthenticationPositive() throws Exception { // test server response to CONNECT ByteBuf clientCommand = Commands.newConnect(authMethodName, "pass.client", null); + BinaryAuthSession binaryAuthSession = + spyBinaryAuthSession(authenticationService, clientCommand.copy(), svcConfig); + when(authenticationService.createBinaryAuthSession(any())).thenReturn(binaryAuthSession); channel.writeInbound(clientCommand); assertTrue(getResponse() instanceof CommandConnected); @@ -483,6 +488,9 @@ public void testConnectCommandWithoutOriginalAuthInfoWhenAuthenticateOriginalAut assertEquals(serverCnx.getState(), State.Start); ByteBuf clientCommand = Commands.newConnect(authMethodName, "pass.client", ""); + BinaryAuthSession binaryAuthSession = + spyBinaryAuthSession(authenticationService, clientCommand.copy(), svcConfig); + when(authenticationService.createBinaryAuthSession(any())).thenReturn(binaryAuthSession); channel.writeInbound(clientCommand); Object response1 = getResponse(); @@ -513,6 +521,9 @@ public void testConnectCommandWithPassingOriginalAuthData() throws Exception { ByteBuf clientCommand = Commands.newConnect(authMethodName, "pass.proxy", 1, null, null, "client", "pass.client", authMethodName); + BinaryAuthSession binaryAuthSession = + spyBinaryAuthSession(authenticationService, clientCommand.copy(), svcConfig); + when(authenticationService.createBinaryAuthSession(any())).thenReturn(binaryAuthSession); channel.writeInbound(clientCommand); Object response1 = getResponse(); @@ -528,6 +539,24 @@ public void testConnectCommandWithPassingOriginalAuthData() throws Exception { channel.finish(); } + private BinaryAuthSession spyBinaryAuthSession(AuthenticationService authenticationService, ByteBuf connectCommand, + ServiceConfiguration serviceConfiguration) { + BinaryAuthContext binaryAuthContext = mock(BinaryAuthContext.class); + when(binaryAuthContext.getAuthenticationService()).thenReturn(authenticationService); + when(binaryAuthContext.isAuthenticateOriginalAuthData()).thenReturn( + serviceConfiguration.isAuthenticateOriginalAuthData()); + when(binaryAuthContext.getExecutor()).thenReturn(serverCnx.ctx().executor()); + when(binaryAuthContext.getIsInitialConnectSupplier()).thenReturn(() -> serverCnx.getState() != State.Connected); + BinaryAuthSession binaryAuthSession = spy(new BinaryAuthSession(binaryAuthContext)); + ByteBuf copy = connectCommand.copy(); + BaseCommand cmd = new BaseCommand(); + int cmdSize = (int) copy.readUnsignedInt(); + cmd.parseFrom(copy, cmdSize); + when(binaryAuthContext.getCommandConnect()).thenReturn(cmd.getConnect()); + + return binaryAuthSession; + } + @Test(timeOut = 30000) public void testConnectCommandWithPassingOriginalAuthDataAndSetAnonymousUserRole() throws Exception { AuthenticationService authenticationService = mock(AuthenticationService.class); @@ -551,6 +580,11 @@ public void testConnectCommandWithPassingOriginalAuthDataAndSetAnonymousUserRole // the proxy will use anonymousUserRole to delegate the client's role when connecting. ByteBuf clientCommand = Commands.newConnect(authMethodName, "pass.proxy", 1, null, null, anonymousUserRole, null, null); + + BinaryAuthSession binaryAuthSession = + spyBinaryAuthSession(authenticationService, clientCommand.copy(), svcConfig); + when(authenticationService.createBinaryAuthSession(any())).thenReturn(binaryAuthSession); + channel.writeInbound(clientCommand); Object response1 = getResponse(); @@ -581,6 +615,9 @@ public void testConnectCommandWithPassingOriginalPrincipal() throws Exception { ByteBuf clientCommand = Commands.newConnect(authMethodName, "pass.proxy", 1, null, null, "client", "pass.client", authMethodName); + BinaryAuthSession binaryAuthSession = + spyBinaryAuthSession(authenticationService, clientCommand.copy(), svcConfig); + when(authenticationService.createBinaryAuthSession(any())).thenReturn(binaryAuthSession); channel.writeInbound(clientCommand); Object response1 = getResponse(); @@ -611,6 +648,9 @@ public void testConnectWithNonProxyRoleAndProxyVersion() throws Exception { ByteBuf clientCommand = Commands.newConnect(authMethodName, AuthData.of("pass.pass".getBytes()), 1, null, null, null, null, null, "my-pulsar-proxy", null); + BinaryAuthSession binaryAuthSession = + spyBinaryAuthSession(authenticationService, clientCommand.copy(), svcConfig); + when(authenticationService.createBinaryAuthSession(any())).thenReturn(binaryAuthSession); channel.writeInbound(clientCommand); Object response = getResponse(); assertTrue(response instanceof CommandError); @@ -637,11 +677,16 @@ public void testAuthChallengePrincipalChangeFails() throws Exception { serverCnx.cancelKeepAliveTask(); ByteBuf clientCommand = Commands.newConnect(authMethodName, "pass.client", ""); + BinaryAuthSession binaryAuthSession = + spyBinaryAuthSession(authenticationService, clientCommand.copy(), svcConfig); + when(authenticationService.createBinaryAuthSession(any())).thenReturn(binaryAuthSession); + channel.writeInbound(clientCommand); Object responseConnected = getResponse(); assertTrue(responseConnected instanceof CommandConnected); assertEquals(serverCnx.getState(), State.Connected); + assertEquals(serverCnx.getAuthRole(), "pass.client"); assertEquals(serverCnx.getPrincipal(), "pass.client"); assertTrue(serverCnx.isActive()); @@ -693,6 +738,9 @@ public void testAuthChallengeOriginalPrincipalChangeFails() throws Exception { ByteBuf clientCommand = Commands.newConnect(authMethodName, "pass.proxy", 1, null, null, "pass.client", "pass.client", authMethodName); + BinaryAuthSession binaryAuthSession = + spyBinaryAuthSession(authenticationService, clientCommand.copy(), svcConfig); + when(authenticationService.createBinaryAuthSession(any())).thenReturn(binaryAuthSession); channel.writeInbound(clientCommand); Object responseConnected = getResponse(); @@ -765,6 +813,9 @@ private void verifyAuthRoleAndOriginalPrincipalBehavior(String authMethodName, S ByteBuf clientCommand = Commands.newConnect(authMethodName, authData, 1, null, null, originalPrincipal, null, null); + BinaryAuthSession binaryAuthSession = + spyBinaryAuthSession(brokerService.getAuthenticationService(), clientCommand.copy(), svcConfig); + when(brokerService.getAuthenticationService().createBinaryAuthSession(any())).thenReturn(binaryAuthSession); channel.writeInbound(clientCommand); Object response = getResponse(); @@ -838,6 +889,9 @@ public void testAuthResponseWithFailingAuthData() throws Exception { // Trigger connect command to result in AuthChallenge ByteBuf clientCommand = Commands.newConnect(authMethodName, "challenge.client", "1"); + BinaryAuthSession binaryAuthSession = + spyBinaryAuthSession(authenticationService, clientCommand.copy(), svcConfig); + when(authenticationService.createBinaryAuthSession(any())).thenReturn(binaryAuthSession); channel.writeInbound(clientCommand); Object challenge1 = getResponse(); @@ -932,6 +986,8 @@ public void testVerifyOriginalPrincipalWithAuthDataForwardedFromProxy() throws E // Submit a failing originalPrincipal to show that it is not used at all. ByteBuf connect = Commands.newConnect(authMethodName, proxyRole, "test", "localhost", "fail.fail", clientRole, authMethodName); + BinaryAuthSession binaryAuthSession = spyBinaryAuthSession(authenticationService, connect.copy(), svcConfig); + when(authenticationService.createBinaryAuthSession(any())).thenReturn(binaryAuthSession); channel.writeInbound(connect); Object connectResponse = getResponse(); assertTrue(connectResponse instanceof CommandConnected); @@ -1224,7 +1280,7 @@ private class ClientChannel implements Closeable { 4), serverCnx); public ClientChannel() { - serverCnx.setAuthRole(""); + serverCnx.clearBinaryAuthSession(); } public void close(){ if (channel != null && channel.isActive()) { @@ -1334,6 +1390,10 @@ public void testVerifyOriginalPrincipalWithoutAuthDataForwardedFromProxy() throw String clientRole = "pass.fail"; ByteBuf connect = Commands.newConnect(authMethodName, proxyRole, "test", "localhost", clientRole, null, null); + + BinaryAuthSession binaryAuthSession = spyBinaryAuthSession(authenticationService, connect.copy(), svcConfig); + when(authenticationService.createBinaryAuthSession(any())).thenReturn(binaryAuthSession); + channel.writeInbound(connect); Object connectResponse = getResponse(); assertTrue(connectResponse instanceof CommandConnected); @@ -1448,6 +1508,8 @@ public void testVerifyAuthRoleAndAuthDataFromDirectConnectionBroker() throws Exc // to pass authentication and fail authorization String clientRole = "pass.fail"; ByteBuf connect = Commands.newConnect(authMethodName, clientRole, "test"); + BinaryAuthSession binaryAuthSession = spyBinaryAuthSession(authenticationService, connect.copy(), svcConfig); + when(authenticationService.createBinaryAuthSession(any())).thenReturn(binaryAuthSession); channel.writeInbound(connect); Object connectResponse = getResponse(); @@ -1518,6 +1580,8 @@ public void testRefreshOriginalPrincipalWithAuthDataForwardedFromProxy() throws String clientRole = "pass.client"; ByteBuf connect = Commands.newConnect(authMethodName, proxyRole, "test", "localhost", clientRole, clientRole, authMethodName); + BinaryAuthSession binaryAuthSession = spyBinaryAuthSession(authenticationService, connect.copy(), svcConfig); + when(authenticationService.createBinaryAuthSession(any())).thenReturn(binaryAuthSession); channel.writeInbound(connect); Object connectResponse = getResponse(); assertTrue(connectResponse instanceof CommandConnected); @@ -1537,7 +1601,7 @@ public void testRefreshOriginalPrincipalWithAuthDataForwardedFromProxy() throws AuthData.of(newClientRole.getBytes(StandardCharsets.UTF_8)), 0, "test"); channel.writeInbound(refreshAuth); - assertEquals(serverCnx.getOriginalAuthData().getCommandData(), newClientRole); + assertEquals(serverCnx.getOriginalAuthData().getCommandData(), clientRole); assertEquals(serverCnx.getOriginalAuthState().getAuthRole(), newClientRole); assertEquals(serverCnx.getAuthData().getCommandData(), proxyRole); assertEquals(serverCnx.getAuthRole(), proxyRole); @@ -1624,6 +1688,51 @@ public void testProducerOnNotOwnedTopic() throws Exception { channel.finish(); } + private PulsarAuthorizationProvider injectAuth() throws Exception { + svcConfig.setAuthorizationEnabled(true); + AuthorizationService authorizationService = + spyWithClassAndConstructorArgs(AuthorizationService.class, svcConfig, pulsar.getPulsarResources()); + Field providerField = AuthorizationService.class.getDeclaredField("provider"); + providerField.setAccessible(true); + PulsarAuthorizationProvider authorizationProvider = + spyWithClassAndConstructorArgs(PulsarAuthorizationProvider.class, svcConfig, + pulsar.getPulsarResources()); + providerField.set(authorizationService, authorizationProvider); + doReturn(authorizationService).when(brokerService).getAuthorizationService(); + svcConfig.setAuthorizationEnabled(true); + + AuthenticationService authenticationService = mock(AuthenticationService.class); + // use a dummy authentication provider + AuthenticationProvider authenticationProvider = new AuthenticationProvider() { + @Override + public void initialize(ServiceConfiguration config) throws IOException { + + } + + @Override + public String authenticate(AuthenticationDataSource authData) throws AuthenticationException { + return "role"; + } + + @Override + public String getAuthMethodName() { + return "dummy"; + } + + @Override + public void close() throws IOException { + + } + }; + + String authMethodName = authenticationProvider.getAuthMethodName(); + when(brokerService.getAuthenticationService()).thenReturn(authenticationService); + when(authenticationService.getAuthenticationProvider(authMethodName)).thenReturn(authenticationProvider); + svcConfig.setAuthenticationEnabled(true); + + return authorizationProvider; + } + @Test(timeOut = 30000) public void testProducerCommandWithAuthorizationPositive() throws Exception { AuthorizationService authorizationService = mock(AuthorizationService.class); @@ -1652,32 +1761,27 @@ public void testProducerCommandWithAuthorizationPositive() throws Exception { @Test(timeOut = 30000) public void testNonExistentTopic() throws Exception { - AuthorizationService authorizationService = - spyWithClassAndConstructorArgs(AuthorizationService.class, svcConfig, pulsar.getPulsarResources()); - doReturn(authorizationService).when(brokerService).getAuthorizationService(); - svcConfig.setAuthorizationEnabled(true); - svcConfig.setAuthorizationEnabled(true); - Field providerField = AuthorizationService.class.getDeclaredField("provider"); - providerField.setAccessible(true); - PulsarAuthorizationProvider authorizationProvider = - spyWithClassAndConstructorArgs(PulsarAuthorizationProvider.class, svcConfig, - pulsar.getPulsarResources()); - providerField.set(authorizationService, authorizationProvider); + resetChannel(); + PulsarAuthorizationProvider authorizationProvider = injectAuth(); doReturn(CompletableFuture.completedFuture(false)).when(authorizationProvider) .isSuperUser(Mockito.anyString(), Mockito.any(), Mockito.any()); + // Connect + ByteBuf connectCommand = Commands.newConnect("dummy", "", null); + BinaryAuthSession binaryAuthSession = + spyBinaryAuthSession(brokerService.getAuthenticationService(), connectCommand.copy(), svcConfig); + when(brokerService.getAuthenticationService().createBinaryAuthSession(any())).thenReturn(binaryAuthSession); + channel.writeInbound(connectCommand); + Object response = getResponse(); + assertTrue(response instanceof CommandConnected); + // Test producer creation - resetChannel(); - setChannelConnected(); ByteBuf newProducerCmd = Commands.newProducer(nonExistentTopicName, 1 /* producer id */, 1 /* request id */, "prod-name", Collections.emptyMap(), false); channel.writeInbound(newProducerCmd); assertTrue(getResponse() instanceof CommandError); - channel.finish(); // Test consumer creation - resetChannel(); - setChannelConnected(); ByteBuf newSubscribeCmd = Commands.newSubscribe(nonExistentTopicName, // successSubName, 1 /* consumer id */, 1 /* request id */, SubType.Exclusive, 0, "test" /* consumer name */, 0); @@ -1688,17 +1792,9 @@ public void testNonExistentTopic() throws Exception { @Test(timeOut = 30000) public void testClusterAccess() throws Exception { - svcConfig.setAuthorizationEnabled(true); - AuthorizationService authorizationService = - spyWithClassAndConstructorArgs(AuthorizationService.class, svcConfig, pulsar.getPulsarResources()); - Field providerField = AuthorizationService.class.getDeclaredField("provider"); - providerField.setAccessible(true); - PulsarAuthorizationProvider authorizationProvider = - spyWithClassAndConstructorArgs(PulsarAuthorizationProvider.class, svcConfig, - pulsar.getPulsarResources()); - providerField.set(authorizationService, authorizationProvider); - doReturn(authorizationService).when(brokerService).getAuthorizationService(); - svcConfig.setAuthorizationEnabled(true); + resetChannel(); + + PulsarAuthorizationProvider authorizationProvider = injectAuth(); doReturn(CompletableFuture.completedFuture(false)).when(authorizationProvider) .isSuperUser(Mockito.anyString(), Mockito.any(), Mockito.any()); doReturn(CompletableFuture.completedFuture(false)).when(authorizationProvider) @@ -1707,15 +1803,20 @@ public void testClusterAccess() throws Exception { .checkPermission(any(TopicName.class), Mockito.anyString(), any(AuthAction.class)); - resetChannel(); - setChannelConnected(); + // Connect + ByteBuf connectCommand = Commands.newConnect("dummy", "", null); + BinaryAuthSession binaryAuthSession = + spyBinaryAuthSession(brokerService.getAuthenticationService(), connectCommand.copy(), svcConfig); + when(brokerService.getAuthenticationService().createBinaryAuthSession(any())).thenReturn(binaryAuthSession); + channel.writeInbound(connectCommand); + Object response = getResponse(); + assertTrue(response instanceof CommandConnected); + ByteBuf clientCommand = Commands.newProducer(successTopicName, 1 /* producer id */, 1 /* request id */, "prod-name", Collections.emptyMap(), false); channel.writeInbound(clientCommand); assertTrue(getResponse() instanceof CommandProducerSuccess); - resetChannel(); - setChannelConnected(); clientCommand = Commands.newProducer(topicWithNonLocalCluster, 1 /* producer id */, 1 /* request id */, "prod-name", Collections.emptyMap(), false); channel.writeInbound(clientCommand); @@ -1725,22 +1826,21 @@ public void testClusterAccess() throws Exception { @Test(timeOut = 30000) public void testNonExistentTopicSuperUserAccess() throws Exception { - AuthorizationService authorizationService = - spyWithClassAndConstructorArgs(AuthorizationService.class, svcConfig, pulsar.getPulsarResources()); - doReturn(authorizationService).when(brokerService).getAuthorizationService(); - svcConfig.setAuthorizationEnabled(true); - Field providerField = AuthorizationService.class.getDeclaredField("provider"); - providerField.setAccessible(true); - PulsarAuthorizationProvider authorizationProvider = - spyWithClassAndConstructorArgs(PulsarAuthorizationProvider.class, svcConfig, - pulsar.getPulsarResources()); - providerField.set(authorizationService, authorizationProvider); + resetChannel(); + PulsarAuthorizationProvider authorizationProvider = injectAuth(); doReturn(CompletableFuture.completedFuture(true)).when(authorizationProvider) .isSuperUser(Mockito.anyString(), Mockito.any(), Mockito.any()); + // Connect + ByteBuf connectCommand = Commands.newConnect("dummy", "", null); + BinaryAuthSession binaryAuthSession = + spyBinaryAuthSession(brokerService.getAuthenticationService(), connectCommand.copy(), svcConfig); + when(brokerService.getAuthenticationService().createBinaryAuthSession(any())).thenReturn(binaryAuthSession); + channel.writeInbound(connectCommand); + Object response = getResponse(); + assertTrue(response instanceof CommandConnected); + // Test producer creation - resetChannel(); - setChannelConnected(); ByteBuf newProducerCmd = Commands.newProducer(nonExistentTopicName, 1 /* producer id */, 1 /* request id */, "prod-name", Collections.emptyMap(), false); channel.writeInbound(newProducerCmd); @@ -1749,11 +1849,8 @@ public void testNonExistentTopicSuperUserAccess() throws Exception { PersistentTopic topicRef = (PersistentTopic) brokerService.getTopicReference(nonExistentTopicName).get(); assertNotNull(topicRef); assertEquals(topicRef.getProducers().size(), 1); - channel.finish(); // Test consumer creation - resetChannel(); - setChannelConnected(); ByteBuf newSubscribeCmd = Commands.newSubscribe(nonExistentTopicName, // successSubName, 1 /* consumer id */, 1 /* request id */, SubType.Exclusive, 0, "test" /* consumer name */, 0 /* avoid reseting cursor */); @@ -2859,7 +2956,6 @@ protected void resetChannel() throws Exception { channel.close().get(); } serverCnx = new ServerCnx(pulsar); - serverCnx.setAuthRole(""); channel = new EmbeddedChannel(new LengthFieldBasedFrameDecoder( maxMessageSize, 0, @@ -3419,7 +3515,7 @@ public boolean isCompletedExceptionally() { @Test public void testHandleAuthResponseWithoutClientVersion() throws Exception { - resetChannel(); + AuthenticationService authenticationService = mock(AuthenticationService.class); // use a dummy authentication provider AuthenticationProvider authenticationProvider = new AuthenticationProvider() { @Override @@ -3442,18 +3538,36 @@ public void close() throws IOException { } }; + + String authMethodName = authenticationProvider.getAuthMethodName(); + when(brokerService.getAuthenticationService()).thenReturn(authenticationService); + when(authenticationService.getAuthenticationProvider(authMethodName)).thenReturn(authenticationProvider); + svcConfig.setAuthenticationEnabled(true); + + resetChannel(); + + ByteBuf clientCommand = Commands.newConnect(authenticationProvider.getAuthMethodName(), "", null); + BinaryAuthSession binaryAuthSession = + spyBinaryAuthSession(authenticationService, clientCommand.copy(), svcConfig); + when(authenticationService.createBinaryAuthSession(any())).thenReturn(binaryAuthSession); + channel.writeInbound(clientCommand); + + // verify that authChallenge is called + Awaitility.await().untilAsserted(() -> verify(binaryAuthSession, times(1)) + .authChallenge(any(), anyBoolean(), anyInt(), any())); + Object response = getResponse(); + assertTrue(response instanceof CommandConnected); + AuthData clientData = AuthData.of(new byte[0]); - AuthenticationState authenticationState = - authenticationProvider.newAuthState(clientData, null, null); - // inject the AuthenticationState instance so that auth response can be processed - serverCnx.setAuthState(authenticationState); // send the auth response with no client version String clientVersion = null; ByteBuf authResponse = - Commands.newAuthResponse("token", clientData, Commands.getCurrentProtocolVersion(), clientVersion); + Commands.newAuthResponse(authenticationProvider.getAuthMethodName(), clientData, + Commands.getCurrentProtocolVersion(), clientVersion); channel.writeInbound(authResponse); - CommandConnected response = (CommandConnected) getResponse(); - assertNotNull(response); + // verify that authChallenge is called again + Awaitility.await().untilAsserted(() -> verify(binaryAuthSession, times(2)) + .authChallenge(any(), anyBoolean(), anyInt(), any())); } @Test(expectedExceptions = IllegalArgumentException.class) @@ -3700,6 +3814,10 @@ public void sendAddPartitionToTxnResponseFailedAuth() throws Exception { ByteBuf connect = Commands.newConnect(authMethodName, "pass.fail", "test", "localhost", "pass.pass", "pass.pass", authMethodName); + + BinaryAuthSession binaryAuthSession = spyBinaryAuthSession(authenticationService, connect.copy(), svcConfig); + when(authenticationService.createBinaryAuthSession(any())).thenReturn(binaryAuthSession); + channel.writeInbound(connect); Object connectResponse = getResponse(); assertTrue(connectResponse instanceof CommandConnected); diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java index f707abbc06af6..ccecce510b66c 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java @@ -43,6 +43,7 @@ import io.netty.handler.timeout.ReadTimeoutHandler; import io.netty.util.CharsetUtil; import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -50,6 +51,9 @@ import lombok.Getter; import lombok.SneakyThrows; import org.apache.pulsar.PulsarVersion; +import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationState; +import org.apache.pulsar.broker.authentication.BinaryAuthSession; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.AuthenticationDataProvider; import org.apache.pulsar.client.api.PulsarClientException; @@ -100,9 +104,28 @@ public DirectProxyHandler(ProxyService service, ProxyConnection proxyConnection) this.inboundChannel = proxyConnection.ctx().channel(); this.proxyConnection = proxyConnection; this.inboundChannelRequestsRate = new Rate(); - this.originalPrincipal = proxyConnection.clientAuthRole; - this.clientAuthData = proxyConnection.clientAuthData; - this.clientAuthMethod = proxyConnection.clientAuthMethod; + BinaryAuthSession binaryAuthSession = proxyConnection.getBinaryAuthSession(); + if (binaryAuthSession != null) { + AuthenticationState originalAuthState = binaryAuthSession.getOriginalAuthState(); + boolean forwardOriginal = + originalAuthState != null && service.getConfiguration().isForwardAuthorizationCredentials(); + AuthenticationDataSource authDataSource = forwardOriginal ? binaryAuthSession.getOriginalAuthData() : + binaryAuthSession.getAuthenticationData(); + String commandData = authDataSource.getCommandData(); + if (commandData != null) { + clientAuthData = AuthData.of(commandData.getBytes(StandardCharsets.UTF_8)); + } else { + clientAuthData = null; + } + clientAuthMethod = + forwardOriginal ? binaryAuthSession.getOriginalAuthMethod() : binaryAuthSession.getAuthMethod(); + originalPrincipal = + forwardOriginal ? binaryAuthSession.getOriginalPrincipal() : binaryAuthSession.getAuthRole(); + } else { + originalPrincipal = null; + clientAuthData = null; + clientAuthMethod = null; + } this.tlsEnabledWithBroker = service.getConfiguration().isTlsEnabledWithBroker(); this.tlsHostnameVerificationEnabled = service.getConfiguration().isTlsHostnameVerificationEnabled(); this.onHandshakeCompleteAction = proxyConnection::cancelKeepAliveTask; diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java index 1e8e2fb55e4a4..548cb21f39bf7 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java @@ -21,9 +21,12 @@ import static com.google.common.base.Preconditions.checkArgument; import io.netty.buffer.ByteBuf; import io.netty.channel.EventLoopGroup; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.PulsarVersion; +import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.BinaryAuthSession; import org.apache.pulsar.client.impl.ClientCnx; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.client.impl.metrics.InstrumentProvider; @@ -41,33 +44,58 @@ */ public class ProxyClientCnx extends ClientCnx { private final boolean forwardClientAuthData; - private final String clientAuthMethod; - private final String clientAuthRole; + private String clientAuthMethod; + private String clientAuthRole; + private final ProxyConnection proxyConnection; + private final BinaryAuthSession binaryAuthSession; - public ProxyClientCnx(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, String clientAuthRole, - String clientAuthMethod, int protocolVersion, + public ProxyClientCnx(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, int protocolVersion, boolean forwardClientAuthData, ProxyConnection proxyConnection) { super(InstrumentProvider.NOOP, conf, eventLoopGroup, protocolVersion); this.clientAuthRole = clientAuthRole; this.clientAuthMethod = clientAuthMethod; this.forwardClientAuthData = forwardClientAuthData; this.proxyConnection = proxyConnection; + this.binaryAuthSession = proxyConnection.getBinaryAuthSession(); } @Override protected ByteBuf newConnectCommand() throws Exception { + AuthData clientAuthData = null; + if (binaryAuthSession != null) { + clientAuthRole = binaryAuthSession.getAuthRole(); + clientAuthMethod = binaryAuthSession.getAuthMethod(); + if (forwardClientAuthData) { + // There is a chance this auth data is expired because the ProxyConnection does not do early token + // refresh. + // Based on the current design, the best option is to configure the broker to accept slightly stale + // authentication data. + String commandData = binaryAuthSession.getAuthenticationData().getCommandData(); + if (commandData != null) { + clientAuthData = AuthData.of(commandData.getBytes(StandardCharsets.UTF_8)); + } + } + + // If original principal is null, it means the client connects the broker via proxy, else the client + // connects the proxy via proxy. + if (binaryAuthSession.getOriginalPrincipal() != null) { + clientAuthRole = binaryAuthSession.getOriginalPrincipal(); + clientAuthMethod = binaryAuthSession.getOriginalAuthMethod(); + AuthenticationDataSource originalAuthData = binaryAuthSession.getOriginalAuthData(); + if (forwardClientAuthData && originalAuthData != null) { + String commandData = originalAuthData.getCommandData(); + if (commandData != null) { + clientAuthData = AuthData.of(commandData.getBytes(StandardCharsets.UTF_8)); + } + } + } + } + if (log.isDebugEnabled()) { log.debug("New Connection opened via ProxyClientCnx with params clientAuthRole = {}," + " clientAuthData = {}, clientAuthMethod = {}", - clientAuthRole, proxyConnection.getClientAuthData(), clientAuthMethod); - } - AuthData clientAuthData = null; - if (forwardClientAuthData) { - // There is a chance this auth data is expired because the ProxyConnection does not do early token refresh. - // Based on the current design, the best option is to configure the broker to accept slightly stale - // authentication data. - clientAuthData = proxyConnection.getClientAuthData(); + clientAuthRole, clientAuthData, clientAuthMethod); } authenticationDataProvider = authentication.getAuthData(remoteHostName); AuthData authData = authenticationDataProvider.authenticate(AuthData.INIT_AUTH_DATA); diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java index cee0a2ca47d57..35d35387c0682 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java @@ -960,6 +960,12 @@ public class ProxyConfiguration implements PulsarConfiguration { ) private String clusterName; + @FieldContext( + category = CATEGORY_AUTHENTICATION, + doc = "If this flag is set then the broker authenticates the original Auth data" + + " else it just accepts the originalPrincipal and authorizes it (if required)") + private boolean authenticateOriginalAuthData = false; + public String getMetadataStoreUrl() { if (StringUtils.isNotBlank(metadataStoreUrl)) { return metadataStoreUrl; diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java index 2512f8af2beec..59728b887dff2 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java @@ -33,6 +33,7 @@ import java.net.InetSocketAddress; import java.net.SocketAddress; import java.nio.channels.ClosedChannelException; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -49,8 +50,10 @@ import javax.net.ssl.SSLSession; import lombok.Getter; import org.apache.pulsar.broker.PulsarServerException; -import org.apache.pulsar.broker.authentication.AuthenticationProvider; import org.apache.pulsar.broker.authentication.AuthenticationState; +import org.apache.pulsar.broker.authentication.BinaryAuthContext; +import org.apache.pulsar.broker.authentication.BinaryAuthSession; +import org.apache.pulsar.broker.authentication.BinaryAuthSession.AuthResult; import org.apache.pulsar.broker.limiter.ConnectionController; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.PulsarClientException; @@ -107,14 +110,8 @@ public class ProxyConnection extends PulsarHandler { private Set> pendingBrokerAuthChallenges = null; private final BrokerProxyValidator brokerProxyValidator; private final ConnectionController connectionController; - String clientAuthRole; - volatile AuthData clientAuthData; - String clientAuthMethod; String clientVersion; - private String authMethod = "none"; - AuthenticationProvider authenticationProvider; - AuthenticationState authState; private ClientConfigurationData clientConf; private boolean hasProxyToBrokerUrl; private int protocolVersionToAdvertise; @@ -122,9 +119,10 @@ public class ProxyConnection extends PulsarHandler { private HAProxyMessage haProxyMessage; protected static final Integer SPLICE_BYTES = 1024 * 1024 * 1024; - private static final byte[] EMPTY_CREDENTIALS = new byte[0]; boolean isTlsInboundChannel = false; + @Getter + private BinaryAuthSession binaryAuthSession; enum State { Init, @@ -348,16 +346,40 @@ protected static boolean isTlsChannel(Channel channel) { private synchronized void completeConnect() throws PulsarClientException { checkArgument(state == State.Connecting); - LOG.info("[{}] complete connection, init proxy handler. authenticated with {} role {}, hasProxyToBrokerUrl: {}", - remoteAddress, authMethod, clientAuthRole, hasProxyToBrokerUrl); + String clientAuthRole; + String clientAuthMethod; + String originalAuthRole; + String originalAuthMethod; + if (binaryAuthSession != null) { + clientAuthRole = binaryAuthSession.getAuthRole(); + clientAuthMethod = binaryAuthSession.getAuthMethod(); + originalAuthRole = binaryAuthSession.getOriginalPrincipal(); + originalAuthMethod = binaryAuthSession.getOriginalAuthMethod(); + } else { + clientAuthRole = null; + clientAuthMethod = null; + originalAuthRole = null; + originalAuthMethod = null; + } + String maybeAnonymizedOriginalAuthRole = originalAuthRole; + LOG.info("[{}] complete connection, init proxy handler. client authenticated with {} role {}, original " + + "authenticated with {} role {}, hasProxyToBrokerUrl: {}", + remoteAddress, + clientAuthMethod, + clientAuthRole, + originalAuthMethod, + originalAuthRole, + hasProxyToBrokerUrl); if (hasProxyToBrokerUrl) { // Optimize proxy connection to fail-fast if the target broker isn't active // Pulsar client will retry connecting after a back off timeout if (service.getConfiguration().isCheckActiveBrokers() && !isBrokerActive(proxyToBrokerUrl)) { state = State.Closing; - LOG.warn("[{}] Target broker '{}' isn't available. authenticated with {} role {}.", - remoteAddress, proxyToBrokerUrl, authMethod, clientAuthRole); + LOG.warn("[{}] Target broker '{}' isn't available. client authenticated with {} role {}, " + + "original authenticated with {} role {}.", + remoteAddress, proxyToBrokerUrl, clientAuthMethod, clientAuthRole, + originalAuthMethod, originalAuthRole); final ByteBuf msg = Commands.newError(-1, ServerError.ServiceNotReady, "Target broker isn't available."); writeAndFlushAndClose(msg); @@ -373,13 +395,18 @@ private synchronized void completeConnect() throws PulsarClientException { TargetAddressDeniedException targetAddressDeniedException = (TargetAddressDeniedException) (throwable instanceof TargetAddressDeniedException ? throwable : throwable.getCause()); - - LOG.warn("[{}] Target broker '{}' cannot be validated. {}. authenticated with {} role {}.", + LOG.warn( + "[{}] Target broker '{}' cannot be validated. {}. client authenticated with {} " + + "role {}, original authenticated with {} role {}.", remoteAddress, proxyToBrokerUrl, targetAddressDeniedException.getMessage(), - authMethod, clientAuthRole); + clientAuthMethod, clientAuthRole, originalAuthMethod, + originalAuthRole); } else { - LOG.error("[{}] Error validating target broker '{}'. authenticated with {} role {}.", - remoteAddress, proxyToBrokerUrl, authMethod, clientAuthRole, throwable); + LOG.error("[{}] Error validating target broker '{}'. client authenticated with {} role {}, " + + "original authenticated with {} role {}.", + remoteAddress, proxyToBrokerUrl, clientAuthMethod, clientAuthRole, + originalAuthMethod, + originalAuthRole, throwable); } final ByteBuf msg = Commands.newError(-1, ServerError.ServiceNotReady, "Target broker cannot be validated."); @@ -391,9 +418,9 @@ private synchronized void completeConnect() throws PulsarClientException { // and we'll take care of just topics and partitions metadata lookups Supplier clientCnxSupplier; if (service.getConfiguration().isAuthenticationEnabled()) { - clientCnxSupplier = () -> new ProxyClientCnx(clientConf, service.getWorkerGroup(), clientAuthRole, - clientAuthMethod, protocolVersionToAdvertise, - service.getConfiguration().isForwardAuthorizationCredentials(), this); + clientCnxSupplier = + () -> new ProxyClientCnx(clientConf, service.getWorkerGroup(), protocolVersionToAdvertise, + service.getConfiguration().isForwardAuthorizationCredentials(), this); } else { clientCnxSupplier = () -> new ClientCnx(InstrumentProvider.NOOP, clientConf, service.getWorkerGroup(), @@ -461,57 +488,12 @@ public void brokerConnected(DirectProxyHandler directProxyHandler, CommandConnec } } - // According to auth result, send newConnected or newAuthChallenge command. - private void doAuthentication(AuthData clientData) - throws Exception { - authState - .authenticateAsync(clientData) - .whenCompleteAsync((authChallenge, throwable) -> { - if (throwable == null) { - authChallengeSuccessCallback(authChallenge); - } else { - authenticationFailedCallback(throwable); - } - }, ctx.executor()); - } - protected void authenticationFailedCallback(Throwable t) { LOG.warn("[{}] Unable to authenticate: ", remoteAddress, t); final ByteBuf msg = Commands.newError(-1, ServerError.AuthenticationError, "Failed to authenticate"); writeAndFlushAndClose(msg); } - // Always run in this class's event loop. - protected void authChallengeSuccessCallback(AuthData authChallenge) { - try { - // authentication has completed, will send newConnected command. - if (authChallenge == null) { - clientAuthRole = authState.getAuthRole(); - if (LOG.isDebugEnabled()) { - LOG.debug("[{}] Client successfully authenticated with {} role {}", - remoteAddress, authMethod, clientAuthRole); - } - - // First connection - if (state == State.Connecting) { - // authentication has completed, will send newConnected command. - completeConnect(); - } - return; - } - - // auth not complete, continue auth with client side. - final ByteBuf msg = Commands.newAuthChallenge(authMethod, authChallenge, protocolVersionToAdvertise); - writeAndFlush(msg); - if (LOG.isDebugEnabled()) { - LOG.debug("[{}] Authentication in progress client by method {}.", - remoteAddress, authMethod); - } - } catch (Exception e) { - authenticationFailedCallback(e); - } - } - private void startAuthRefreshTaskIfNotStarted() { if (service.getConfiguration().isAuthenticationEnabled() && service.getConfiguration().getAuthenticationRefreshCheckSeconds() > 0 @@ -531,9 +513,7 @@ private void refreshAuthenticationCredentialsAndCloseIfTooExpired() { // Only check expiration in authenticated states if (!state.isAuthenticatedState()) { return; - } - - if (!authState.isExpired()) { + } else if (!binaryAuthSession.isExpired()) { // Credentials are still valid. Nothing to do at this point return; } @@ -562,8 +542,7 @@ private void refreshAuthenticationCredentialsAndCloseIfTooExpired() { private void maybeSendAuthChallenge() { assert ctx.executor().inEventLoop(); - if (!supportsAuthenticationRefresh()) { - LOG.warn("[{}] Closing connection because client doesn't support auth credentials refresh", remoteAddress); + if (!binaryAuthSession.supportsAuthenticationRefresh()) { ctx.close(); return; } else if (authChallengeSentTime != Long.MAX_VALUE) { @@ -582,11 +561,12 @@ private void maybeSendAuthChallenge() { LOG.debug("[{}] Refreshing authentication credentials", remoteAddress); } try { - AuthData challenge = authState.refreshAuthentication(); - writeAndFlush(Commands.newAuthChallenge(authMethod, challenge, protocolVersionToAdvertise)); + AuthResult authResult = binaryAuthSession.refreshAuthentication(); + writeAndFlush(Commands.newAuthChallenge(authResult.getAuthMethod(), authResult.getAuthData(), + protocolVersionToAdvertise)); if (LOG.isDebugEnabled()) { LOG.debug("[{}] Sent auth challenge to client to refresh credentials with method: {}.", - remoteAddress, authMethod); + remoteAddress, authResult.getAuthMethod()); } authChallengeSentTime = System.nanoTime(); } catch (AuthenticationException e) { @@ -624,15 +604,6 @@ remoteAddress, protocolVersionToAdvertise, getRemoteEndpointProtocolVersion(), return; } - if (connect.hasProxyVersion()) { - if (LOG.isDebugEnabled()) { - LOG.debug("[{}] Client illegally provided proxyVersion.", remoteAddress); - } - state = State.Closing; - writeAndFlushAndClose(Commands.newError(-1, ServerError.NotAllowedError, "Must not provide proxyVersion")); - return; - } - try { // init authn this.clientConf = createClientConfiguration(); @@ -643,39 +614,6 @@ remoteAddress, protocolVersionToAdvertise, getRemoteEndpointProtocolVersion(), return; } - AuthData clientData = AuthData.of(connect.hasAuthData() ? connect.getAuthData() : EMPTY_CREDENTIALS); - if (connect.hasAuthMethodName()) { - authMethod = connect.getAuthMethodName(); - } else if (connect.hasAuthMethod()) { - // Legacy client is passing enum - authMethod = connect.getAuthMethod().name().substring(10).toLowerCase(); - } else { - authMethod = "none"; - } - - if (service.getConfiguration().isForwardAuthorizationCredentials()) { - // We store the first clientData here. Before this commit, we stored the last clientData. - // Since this only works when forwarding single staged authentication, first == last is true. - // Here is an issue to fix the protocol: https://github.com/apache/pulsar/issues/19291. - this.clientAuthData = clientData; - this.clientAuthMethod = authMethod; - } - - authenticationProvider = service - .getAuthenticationService() - .getAuthenticationProvider(authMethod); - - // Not find provider named authMethod. Most used for tests. - // In AuthenticationDisabled, it will set authMethod "none". - if (authenticationProvider == null) { - clientAuthRole = service.getAuthenticationService().getAnonymousUserRole() - .orElseThrow(() -> - new AuthenticationException("No anonymous role, and no authentication provider configured")); - - completeConnect(); - return; - } - // init authState and other var ChannelHandler sslHandler = ctx.channel().pipeline().get(PulsarChannelInitializer.TLS_HANDLER); SSLSession sslSession = null; @@ -683,8 +621,59 @@ remoteAddress, protocolVersionToAdvertise, getRemoteEndpointProtocolVersion(), sslSession = ((SslHandler) sslHandler).engine().getSession(); } - authState = authenticationProvider.newAuthState(clientData, remoteAddress, sslSession); - doAuthentication(clientData); + binaryAuthSession = service.getAuthenticationService().createBinaryAuthSession(BinaryAuthContext.builder() + .executor(ctx.executor()) + .remoteAddress(remoteAddress) + .sslSession(sslSession) + .authenticationService(service.getAuthenticationService()) + .commandConnect(connect) + .isInitialConnectSupplier(() -> state == State.Connecting) + .authenticateOriginalAuthData(service.getConfiguration().isAuthenticateOriginalAuthData()) + .build()); + binaryAuthSession.doAuthentication() + .whenCompleteAsync((authResult, ex) -> { + if (ex != null) { + authenticationFailedCallback(ex); + } else { + handleAuthResult(authResult); + } + }, ctx.executor()); + } catch (Exception e) { + authenticationFailedCallback(e); + } + } + + private void handleAuthResult(BinaryAuthSession.AuthResult authResult) { + try { + AuthData authData = authResult.getAuthData(); + if (authData != null) { + writeAndFlush(Commands.newAuthChallenge( + authResult.getAuthMethod(), + authData, + protocolVersionToAdvertise)); + if (LOG.isDebugEnabled()) { + LOG.debug("[{}] Authentication in progress client by method {}.", remoteAddress, + authResult.getAuthMethod()); + } + } else { + // First connection + if (state == State.Connecting) { + // authentication has completed, will send newConnected command. + completeConnect(); + } else { + if (service.getConfiguration().isForwardAuthorizationCredentials()) { + // We only have pendingBrokerAuthChallenges when forwardAuthorizationCredentials is enabled. + if (pendingBrokerAuthChallenges != null && !pendingBrokerAuthChallenges.isEmpty()) { + // Send auth data to pending challenges from the broker + for (CompletableFuture challenge : pendingBrokerAuthChallenges) { + challenge.complete(AuthData.of(getFinalAuthState().getAuthDataSource().getCommandData() + .getBytes(StandardCharsets.UTF_8))); + } + pendingBrokerAuthChallenges.clear(); + } + } + } + } } catch (Exception e) { authenticationFailedCallback(e); } @@ -700,6 +689,12 @@ protected void handleAuthResponse(CommandAuthResponse authResponse) { remoteAddress, authResponse.getResponse().getAuthMethodName()); } + if (binaryAuthSession == null) { + authenticationFailedCallback( + new AuthenticationException("Authentication session is null or not initialized")); + return; + } + try { // Reset the auth challenge sent time to indicate we are not waiting on a client response. authChallengeSentTime = Long.MAX_VALUE; @@ -708,24 +703,16 @@ protected void handleAuthResponse(CommandAuthResponse authResponse) { // Note: this implementation relies on the current weakness that prevents multi-stage authentication // from working when forwardAuthorizationCredentials is enabled. Here is an issue to fix the protocol: // https://github.com/apache/pulsar/issues/19291. - doAuthentication(clientData); - if (service.getConfiguration().isForwardAuthorizationCredentials()) { - // Update the clientAuthData to be able to initialize future ProxyClientCnx. - this.clientAuthData = clientData; - // We only have pendingBrokerAuthChallenges when forwardAuthorizationCredentials is enabled. - if (pendingBrokerAuthChallenges != null && !pendingBrokerAuthChallenges.isEmpty()) { - // Send auth data to pending challenges from the broker - for (CompletableFuture challenge : pendingBrokerAuthChallenges) { - challenge.complete(clientData); - } - pendingBrokerAuthChallenges.clear(); - } - } + binaryAuthSession.authChallenge(clientData, false, protocolVersionToAdvertise, clientVersion) + .whenCompleteAsync((authResult, ex) -> { + if (ex != null) { + authenticationFailedCallback(ex); + } else { + handleAuthResult(authResult); + } + }, ctx.executor()); } catch (Exception e) { - String errorMsg = "Unable to handleAuthResponse"; - LOG.warn("[{}] {} ", remoteAddress, errorMsg, e); - final ByteBuf msg = Commands.newError(-1, ServerError.AuthenticationError, errorMsg); - writeAndFlushAndClose(msg); + authenticationFailedCallback(e); } } @@ -869,12 +856,10 @@ private void writeAndFlushAndClose(ByteBuf cmd) { NettyChannelUtil.writeAndFlushWithClosePromise(ctx, cmd); } - boolean supportsAuthenticationRefresh() { - return features != null && features.isSupportsAuthRefresh(); - } - - AuthData getClientAuthData() { - return clientAuthData; + private AuthenticationState getFinalAuthState() { + AuthenticationState originalAuthState = binaryAuthSession.getOriginalAuthState(); + AuthenticationState authState = binaryAuthSession.getAuthState(); + return originalAuthState != null ? originalAuthState : authState; } /** @@ -884,9 +869,11 @@ AuthData getClientAuthData() { CompletableFuture getValidClientAuthData() { final CompletableFuture clientAuthDataFuture = new CompletableFuture<>(); ctx().executor().execute(Runnables.catchingAndLoggingThrowables(() -> { + AuthenticationState finalAuthState = getFinalAuthState(); // authState is not thread safe, so this must run on the ProxyConnection's event loop. - if (!authState.isExpired()) { - clientAuthDataFuture.complete(clientAuthData); + if (!finalAuthState.isExpired()) { + clientAuthDataFuture.complete(AuthData.of( + finalAuthState.getAuthDataSource().getCommandData().getBytes(StandardCharsets.UTF_8))); } else if (state == State.ProxyLookupRequests) { maybeSendAuthChallenge(); if (pendingBrokerAuthChallenges == null) { diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java index 04529629de7f1..bc874fbbe554e 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java @@ -318,7 +318,7 @@ void testAuthentication() throws Exception { // Step 4: Ensure that all client contexts share the same auth provider Assert.assertTrue(proxyService.getClientCnxs().size() >= 3, "expect at least 3 clients"); proxyService.getClientCnxs().stream().forEach((cnx) -> { - Assert.assertSame(cnx.authenticationProvider, + Assert.assertSame(cnx.getBinaryAuthSession().getAuthenticationProvider(), proxyService.getAuthenticationService().getAuthenticationProvider("BasicAuthentication")); }); } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyToProxyAuthenticationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyToProxyAuthenticationTest.java new file mode 100644 index 0000000000000..5fc0b502311bd --- /dev/null +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyToProxyAuthenticationTest.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.proxy.server; + +import static org.assertj.core.api.Assertions.assertThat; +import io.jsonwebtoken.Jwts; +import io.jsonwebtoken.SignatureAlgorithm; +import java.util.Base64; +import java.util.Collections; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import javax.crypto.SecretKey; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; +import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; +import org.apache.pulsar.broker.service.ServerCnx; +import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; +import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; +import org.apache.pulsar.common.naming.TopicName; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Slf4j +public class ProxyToProxyAuthenticationTest extends ProducerConsumerBase { + private static final String CLUSTER_NAME = "test"; + + private static final String ADMIN_ROLE = "admin"; + private static final String PROXY_ROLE = "proxy"; + private static final String BROKER_ROLE = "broker"; + private static final String CLIENT_ROLE = "client"; + private static final SecretKey SECRET_KEY = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); + + private static final String ADMIN_TOKEN = Jwts.builder().setSubject(ADMIN_ROLE).signWith(SECRET_KEY).compact(); + private static final String PROXY_TOKEN = Jwts.builder().setSubject(PROXY_ROLE).signWith(SECRET_KEY).compact(); + private static final String BROKER_TOKEN = Jwts.builder().setSubject(BROKER_ROLE).signWith(SECRET_KEY).compact(); + private static final String CLIENT_TOKEN = Jwts.builder().setSubject(CLIENT_ROLE).signWith(SECRET_KEY).compact(); + + @BeforeMethod + @Override + protected void setup() throws Exception { + conf.setAuthenticateOriginalAuthData(false); + conf.setAuthenticationEnabled(true); + conf.getProperties().setProperty("tokenSecretKey", "data:;base64," + + Base64.getEncoder().encodeToString(SECRET_KEY.getEncoded())); + + Set superUserRoles = new HashSet<>(); + superUserRoles.add(ADMIN_ROLE); + superUserRoles.add(PROXY_ROLE); + superUserRoles.add(BROKER_ROLE); + conf.setSuperUserRoles(superUserRoles); + conf.setProxyRoles(Collections.singleton(PROXY_ROLE)); + + conf.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); + conf.setBrokerClientAuthenticationParameters(BROKER_TOKEN); + Set providers = new HashSet<>(); + providers.add(AuthenticationProviderToken.class.getName()); + conf.setAuthenticationProviders(providers); + + conf.setClusterName(CLUSTER_NAME); + super.init(); + + admin = PulsarAdmin.builder().serviceHttpUrl(pulsar.getWebServiceAddress()).authentication( + AuthenticationFactory.token(ADMIN_TOKEN)).build(); + producerBaseSetup(); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + private ProxyService createProxyService(String serviceUrl) throws Exception { + ProxyConfiguration proxyConfig = new ProxyConfiguration(); + proxyConfig.setForwardAuthorizationCredentials(true); + proxyConfig.setAuthenticateOriginalAuthData(true); + proxyConfig.setAuthenticationEnabled(true); + proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); + proxyConfig.setWebServicePort(Optional.of(0)); + proxyConfig.setBrokerServiceURL(serviceUrl); + proxyConfig.setClusterName(CLUSTER_NAME); + + proxyConfig.getProperties().setProperty("tokenSecretKey", "data:;base64," + + Base64.getEncoder().encodeToString(SECRET_KEY.getEncoded())); + + proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); + proxyConfig.setBrokerClientAuthenticationParameters(PROXY_TOKEN); + + Set providers = new HashSet<>(); + providers.add(AuthenticationProviderToken.class.getName()); + proxyConfig.setAuthenticationProviders(providers); + AuthenticationService authenticationService = new AuthenticationService( + PulsarConfigurationLoader.convertFrom(proxyConfig)); + Authentication proxyClientAuthentication = + AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); + ProxyService proxyService = new ProxyService(proxyConfig, authenticationService, proxyClientAuthentication); + proxyService.start(); + return proxyService; + } + + @Test + public void testClientConnectsThroughTwoAuthenticatedProxiesToBroker() throws Exception { + @Cleanup + ProxyService az1Proxy = createProxyService(pulsar.getBrokerServiceUrl()); + String az1ProxyServiceUrl = az1Proxy.getServiceUrl(); + @Cleanup + ProxyService az2Proxy = createProxyService(az1ProxyServiceUrl); + @Cleanup + PulsarClient pulsarClient = + PulsarClient.builder().serviceUrl(az2Proxy.getServiceUrl()) + .authentication(AuthenticationFactory.token(CLIENT_TOKEN)) + .build(); + String topic = TopicName.get("test-topic").toString(); + String subscription1 = "test-subscription"; + @Cleanup + Consumer consumer1 = + pulsarClient.newConsumer().topic(topic).subscriptionName(subscription1).subscribe(); + String subscription2 = "test2-subscription"; + @Cleanup + Consumer consumer2 = + pulsarClient.newConsumer().topic(topic).subscriptionName(subscription2).subscribe(); + + CompletableFuture> topicIfExists = pulsar.getBrokerService().getTopicIfExists(topic); + assertThat(topicIfExists).succeedsWithin(3, TimeUnit.SECONDS); + Topic topicRef = topicIfExists.get().orElseThrow(); + topicRef.getSubscriptions().forEach((key, value) -> { + ServerCnx cnx = (ServerCnx) value.getConsumers().get(0).cnx(); + assertThat(cnx.getAuthRole()).isEqualTo(PROXY_ROLE); + assertThat(cnx.getBinaryAuthSession().getOriginalPrincipal()).isEqualTo(CLIENT_ROLE); + }); + + consumer1.close(); + consumer2.close(); + } +} From 2dedc552be1669f93a2d325555fc76b7f94f5e3b Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Thu, 26 Mar 2026 18:21:01 +0800 Subject: [PATCH 09/16] Fix test --- .../pulsar/broker/service/BrokerService.java | 17 +++-------------- .../resourcegroup/ResourceGroupServiceTest.java | 1 + ...ceGroupUsageAggregationOnTopicLevelTest.java | 1 + .../nonpersistent/NonPersistentTopicTest.java | 2 +- 4 files changed, 6 insertions(+), 15 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 185c8b9fff93d..e23b051db85f4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1550,20 +1550,9 @@ private CompletableFuture> createNonPersistentTopic(TopicLoading return null; }); }).exceptionally(e -> { - log.warn("CheckTopicNsOwnership fail when createNonPersistentTopic! {}", topic, e.getCause()); - dispatchEvent.accept(null); - // CheckTopicNsOwnership fail dont create nonPersistentTopic, when topic do lookup will find the - // correct - // broker. When client get non-persistent-partitioned topic - // metadata will the non-persistent-topic will be created. - // so we should add checkTopicNsOwnership logic otherwise the topic will be created - // if it dont own by this broker,we should return success - // otherwise it will keep retrying getPartitionedTopicMetadata - topicFuture.complete(Optional.of(nonPersistentTopic)); - // after get metadata return success, we should delete this topic from this broker, because this - // topic not - // owner by this broker and it don't initialize and checkReplication - pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); + Throwable throwable = FutureUtil.unwrapCompletionException(e); + dispatchEvent.accept(throwable); + topicFuture.completeExceptionally(throwable); return null; }); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java index 2ff3ef351aedb..7c48b78868b3b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java @@ -52,6 +52,7 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +@Test(groups = "flaky") public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest { @BeforeClass @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationOnTopicLevelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationOnTopicLevelTest.java index 3a6c08b8c688f..c950a50a55d8d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationOnTopicLevelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationOnTopicLevelTest.java @@ -46,6 +46,7 @@ import org.testng.annotations.Test; @Slf4j +@Test(groups = "flaky") public class ResourceGroupUsageAggregationOnTopicLevelTest extends ProducerConsumerBase { private final String tenantName = "pulsar-test"; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java index d902434f9bdb6..12b6cd2761a19 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java @@ -116,7 +116,7 @@ public void testCreateNonExistentPartitions() throws PulsarAdminException { final String topicName = "non-persistent://prop/ns-abc/testCreateNonExistentPartitions"; admin.topics().createPartitionedTopic(topicName, 4); TopicName partition = TopicName.get(topicName).getPartition(4); - assertThrows(PulsarClientException.NotAllowedException.class, () -> { + assertThrows(PulsarClientException.NotFoundException.class, () -> { @Cleanup Producer ignored = pulsarClient.newProducer() .topic(partition.toString()) From 86a93660ab43143aee31b4c1786199e7349d6ffb Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 27 Mar 2026 14:14:50 +0800 Subject: [PATCH 10/16] Fix license --- distribution/server/src/assemble/LICENSE.bin.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 82227a2eb7ff6..fb6d0040bd16b 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -580,6 +580,9 @@ Protocol Buffers License CDDL-1.1 -- ../licenses/LICENSE-CDDL-1.1.txt * Java Annotations API - com.sun.activation-jakarta.activation-1.2.2.jar + - javax.activation-javax.activation-api-1.2.0.jar + * Java Architecture for XML Binding (JAXB) API + - javax.xml.bind-jaxb-api-2.3.1.jar * Java Servlet API -- javax.servlet-javax.servlet-api-3.1.0.jar * WebSocket Server API -- javax.websocket-javax.websocket-client-api-1.0.jar * HK2 - Dependency Injection Kernel From 7931624cb2e70c5b0941fa267426a560bd7304e7 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 27 Mar 2026 14:44:54 +0800 Subject: [PATCH 11/16] Fix test --- .../ReplicatedSubscriptionsControllerTest.java | 1 + .../pulsar/client/api/ConsumerCreationTest.java | 15 ++++----------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsControllerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsControllerTest.java index 92a15ad260553..818091ade2d1c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsControllerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsControllerTest.java @@ -71,6 +71,7 @@ public void testSnapshotRequestWhenReplicatorRemovedConcurrentlyDoesNotThrow() t ServiceConfiguration config = new ServiceConfiguration(); config.setReplicatorPrefix("pulsar.repl"); config.setEnableReplicatedSubscriptions(true); + config.setClusterName("test-cluster"); OpenTelemetryReplicatedSubscriptionStats stats = mock(OpenTelemetryReplicatedSubscriptionStats.class); MonotonicClock monotonicClock = System::nanoTime; BacklogQuotaManager backlogQuotaManager = mock(BacklogQuotaManager.class); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerCreationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerCreationTest.java index 195485739e0d6..5fecd46a71bff 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerCreationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerCreationTest.java @@ -112,16 +112,9 @@ private void testCreateConsumerWhenSinglePartitionIsDeleted(TopicDomain domain, admin.topics().delete(TopicName.get(partitionedTopic).getPartition(1).toString()); // Non-persistent topic only have the metadata, and no partition, so it works fine. - if (allowAutoTopicCreation || domain.equals(TopicDomain.non_persistent)) { - @Cleanup - Consumer ignored = - pulsarClient.newConsumer().topic(partitionedTopic).subscriptionName("my-sub").subscribe(); - } else { - assertThrows(PulsarClientException.NotFoundException.class, () -> { - @Cleanup - Consumer ignored = - pulsarClient.newConsumer().topic(partitionedTopic).subscriptionName("my-sub").subscribe(); - }); - } + //When metadata exists, persistent topic will automatically create deleted partitions. + @Cleanup + Consumer ignored = + pulsarClient.newConsumer().topic(partitionedTopic).subscriptionName("my-sub").subscribe(); } } \ No newline at end of file From 162a37a8bccd9678cc8843dfae164653b1960f8b Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 27 Mar 2026 15:17:02 +0800 Subject: [PATCH 12/16] Fix license --- distribution/shell/src/assemble/LICENSE.bin.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 5769ecde6dc23..a0813bac9dbaf 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -433,6 +433,9 @@ MIT License CDDL-1.1 -- ../licenses/LICENSE-CDDL-1.1.txt * Java Annotations API - jakarta.activation-1.2.2.jar + - javax.activation-api-1.2.0.jar + * Java Architecture for XML Binding (JAXB) API + - jaxb-api-2.3.1.jar * WebSocket Server API -- javax.websocket-client-api-1.0.jar * HK2 - Dependency Injection Kernel - hk2-api-2.6.1.jar From 80c2cfbae3c49c05709fbbcc118fc629462f405b Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 27 Mar 2026 15:22:45 +0800 Subject: [PATCH 13/16] Fix test --- .../org/apache/pulsar/client/api/ConsumerCreationTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerCreationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerCreationTest.java index 5fecd46a71bff..788fabf8f47b1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerCreationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerCreationTest.java @@ -23,6 +23,7 @@ import lombok.Cleanup; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.PulsarClientException.NotAllowedException; +import org.apache.pulsar.client.api.PulsarClientException.NotFoundException; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.testng.annotations.AfterMethod; @@ -86,7 +87,7 @@ public void testCreateConsumerWhenTopicTypeMismatch(TopicDomain domain) } // Partition index is out of range. - assertThrows(NotAllowedException.class, () -> { + assertThrows(NotFoundException.class, () -> { @Cleanup Consumer ignored = pulsarClient.newConsumer().topic(TopicName.get(partitionedTopic).getPartition(100).toString()) From feaf9807e7c94e25c3c1541c9a5e32e8a73194e8 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 27 Mar 2026 15:41:56 +0800 Subject: [PATCH 14/16] Fix auth role --- .../apache/pulsar/broker/service/ServerCnx.java | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 2a704c4f8cf75..c4ff3377751fe 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -231,16 +231,6 @@ public class ServerCnx extends PulsarHandler implements TransportCnx { private final BrokerInterceptor brokerInterceptor; private State state; private volatile boolean isActive = true; - private String authRole = null; - private volatile AuthenticationDataSource authenticationData; - private AuthenticationProvider authenticationProvider; - private AuthenticationState authState; - // In case of proxy, if the authentication credentials are forwardable, - // it will hold the credentials of the original client - private AuthenticationState originalAuthState; - private volatile AuthenticationDataSource originalAuthData; - // Keep temporarily in order to verify after verifying proxy's authData - private AuthData originalAuthDataCopy; private boolean pendingAuthChallengeResponse = false; private ScheduledFuture authRefreshTask; @@ -978,7 +968,7 @@ private void authenticationFailed(Throwable t) { private void maybeScheduleAuthenticationCredentialsRefresh() { assert ctx.executor().inEventLoop(); assert authRefreshTask == null; - if (authState == null) { + if (getAuthRole() == null) { // Authentication is disabled or there's no local state to refresh return; } @@ -3678,7 +3668,7 @@ public BrokerService getBrokerService() { } public String getRole() { - return authRole; + return binaryAuthSession != null ? binaryAuthSession.getAuthRole() : null; } @Override From 8d3093a4bf1f57c2e429794eb756cbe0818bf871 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 27 Mar 2026 18:30:34 +0800 Subject: [PATCH 15/16] Fix test --- .../pulsar/broker/service/BrokerServiceTest.java | 4 ++-- .../pulsar/client/api/ProducerCreationTest.java | 15 +++++---------- .../pulsar/websocket/proxy/ProxyRoleAuthTest.java | 2 +- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java index e7af46fc44d35..529584073e99e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java @@ -1185,8 +1185,8 @@ public void testTopicLoadingOnDisableNamespaceBundle() throws Exception { pulsar.getNamespaceService().getOwnershipCache().updateBundleState(bundle, false).join(); // try to create topic which should fail as bundle is disable - TopicLoadingContext topicLoadingContext = - TopicLoadingContext.builder().topicName(topic).createIfMissing(true).properties(null).build(); + TopicLoadingContext topicLoadingContext = TopicLoadingContext.builder().topicFuture(new CompletableFuture<>()) + .topicName(topic).createIfMissing(true).properties(null).build(); CompletableFuture> futureResult = pulsar.getBrokerService() .loadOrCreatePersistentTopic(topicLoadingContext); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerCreationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerCreationTest.java index e13423a213119..abdf8d6f5d1d8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerCreationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerCreationTest.java @@ -23,6 +23,7 @@ import lombok.Cleanup; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.PulsarClientException.NotAllowedException; +import org.apache.pulsar.client.api.PulsarClientException.NotFoundException; import org.apache.pulsar.client.impl.ProducerBuilderImpl; import org.apache.pulsar.client.impl.ProducerImpl; import org.apache.pulsar.common.naming.TopicDomain; @@ -231,7 +232,7 @@ public void testCreateProducerWhenTopicTypeMismatch(TopicDomain domain) } // Partition index is out of range. - assertThrows(NotAllowedException.class, () -> { + assertThrows(NotFoundException.class, () -> { @Cleanup Producer ignored = pulsarClient.newProducer().topic(TopicName.get(partitionedTopic).getPartition(100).toString()) @@ -257,14 +258,8 @@ private void testCreateProducerWhenSinglePartitionIsDeleted(TopicDomain domain, admin.topics().delete(TopicName.get(partitionedTopic).getPartition(1).toString()); // Non-persistent topic only have the metadata, and no partition, so it works fine. - if (allowAutoTopicCreation || domain == TopicDomain.non_persistent) { - @Cleanup - Producer ignored = pulsarClient.newProducer().topic(partitionedTopic).create(); - } else { - assertThrows(PulsarClientException.NotFoundException.class, () -> { - @Cleanup - Producer ignored = pulsarClient.newProducer().topic(partitionedTopic).create(); - }); - } + //When metadata exists, persistent topic will automatically create deleted partitions. + @Cleanup + Producer ignored = pulsarClient.newProducer().topic(partitionedTopic).create(); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyRoleAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyRoleAuthTest.java index 0eb90b9fe363d..46983b150b604 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyRoleAuthTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyRoleAuthTest.java @@ -67,7 +67,7 @@ * 1. testWebSocketProxyProduceConsumeWithAuthorization: Positive test with authorized tokens * 2. testWebSocketProxyWithUnauthorizedToken: Negative test with unauthorized tokens */ -@Test(groups = "websocket") +@Test(groups = "flaky") public class ProxyRoleAuthTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(ProxyRoleAuthTest.class); From a11fc64c568c682124cd3330e513f8e80c4ac521 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 27 Mar 2026 21:48:39 +0800 Subject: [PATCH 16/16] Fix test --- .../pulsar/broker/resourcegroup/ResourceGroupService.java | 3 ++- .../broker/service/persistent/MessageDuplicationTest.java | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java index f11a839f8bf0c..539c5967291d8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java @@ -990,7 +990,8 @@ protected void calculateQuotaByMonClass(String rgName, ResourceGroup resourceGro } private void initialize() { - long resourceUsagePublishPeriodInMS = TimeUnit.SECONDS.toMillis(this.resourceUsagePublishPeriodInSeconds); + long resourceUsagePublishPeriodInMS = + TimeUnit.SECONDS.toMillis(pulsar.getConfiguration().getResourceUsageTransportPublishIntervalInSecs()); long statsCacheInMS = resourceUsagePublishPeriodInMS * 2; topicProduceStats = newStatsCache(statsCacheInMS); topicConsumeStats = newStatsCache(statsCacheInMS); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java index 047a09fcffde8..e80a2f6e99d47 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java @@ -255,6 +255,7 @@ public void testIsDuplicateWithFailure() { serviceConfiguration.setBrokerDeduplicationEntriesInterval(BROKER_DEDUPLICATION_ENTRIES_INTERVAL); serviceConfiguration.setBrokerDeduplicationMaxNumberOfProducers(BROKER_DEDUPLICATION_MAX_NUMBER_PRODUCERS); serviceConfiguration.setReplicatorPrefix(REPLICATOR_PREFIX); + serviceConfiguration.setClusterName("test-cluster"); doReturn(serviceConfiguration).when(pulsarService).getConfiguration(); doReturn(mock(PulsarResources.class)).when(pulsarService).getPulsarResources();