diff --git a/.idea/vcs.xml b/.idea/vcs.xml index 29e5254a1d41b..40d9fc2e26866 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -1,16 +1,16 @@ - + - + - \ No newline at end of file + diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index de4bd7de54d9a..2ac1ab79d0d7b 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -1,4 +1,4 @@ - + - + com.gradle gradle-enterprise-maven-extension diff --git a/.mvn/gradle-enterprise.xml b/.mvn/gradle-enterprise.xml index effb5bb221325..eec67bc7edad0 100644 --- a/.mvn/gradle-enterprise.xml +++ b/.mvn/gradle-enterprise.xml @@ -1,4 +1,4 @@ - + - - #{env['GRADLE_ENTERPRISE_ACCESS_KEY']?.trim() > ''} + + #{env['GRADLE_ENTERPRISE_ACCESS_KEY']?.trim() > ''} https://ge.apache.org false diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index cf83a68be7f75..7b8efd37941c4 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative bouncy-castle-parent 3.2.0-SNAPSHOT .. - bouncy-castle-bc Apache Pulsar :: Bouncy Castle :: BC - ${project.groupId} @@ -39,23 +36,19 @@ ${project.version} provided - org.bouncycastle bcpkix-jdk18on ${bouncycastle.version} - org.bouncycastle bcprov-ext-jdk18on ${bouncycastle.version} - - de.ntcomputer @@ -73,7 +66,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index 56639ff43144e..5548fb92cedf8 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative bouncy-castle-parent 3.2.0-SNAPSHOT .. - bcfips-include-test Pulsar Bouncy Castle FIPS Test Broker and client runs auth include BC FIPS verison - ${project.groupId} @@ -39,7 +37,6 @@ ${project.version} test - ${project.groupId} pulsar-broker @@ -53,7 +50,6 @@ test-jar test - ${project.groupId} pulsar-broker @@ -66,7 +62,6 @@ test - ${project.groupId} @@ -74,7 +69,6 @@ ${project.version} pkg - diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index f38a8e0d2afdb..ec33cdd91fc7b 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative bouncy-castle-parent 3.2.0-SNAPSHOT .. - bouncy-castle-bcfips Apache Pulsar :: Bouncy Castle :: BC-FIPS - ${project.groupId} @@ -39,20 +36,17 @@ ${project.version} provided - org.bouncycastle bc-fips ${bouncycastle.bc-fips.version} - org.bouncycastle bcpkix-fips ${bouncycastle.bcpkix-fips.version} - diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index 3b46eb4a90c9e..c97c0061dc4f2 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - @@ -42,16 +41,13 @@ - bouncy-castle-parent Apache Pulsar :: Bouncy Castle :: Parent - bc bcfips - bcfips-include-test diff --git a/buildtools/pom.xml b/buildtools/pom.xml index c92d68ccc8022..b1cf5e0263e3c 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache apache 29 - + - - org.apache.pulsar + io.streamnative buildtools 3.2.0-SNAPSHOT jar Pulsar Build Tools - - 2023-09-08T03:29:19Z + 2024-03-04T21:08:59Z 1.8 1.8 3.1.0 @@ -60,7 +57,6 @@ --add-opens java.base/jdk.internal.platform=ALL-UNNAMED - @@ -72,9 +68,7 @@ - - org.yaml snakeyaml @@ -95,7 +89,6 @@ guice ${guice.version} - org.testng testng @@ -142,7 +135,6 @@ ${mockito.version} - @@ -251,6 +243,55 @@ **/proto/* + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + + + org.apache.maven.plugins + maven-source-plugin + 3.2.0 + + + attach-sources + + jar-no-fork + + + + + + maven-gpg-plugin + + + sign-artifacts + verify + + sign + + + + @@ -260,4 +301,14 @@ + + + ossrh + https://s01.oss.sonatype.org/content/repositories/snapshots + + + ossrh + https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ + + diff --git a/buildtools/src/main/resources/log4j2.xml b/buildtools/src/main/resources/log4j2.xml index 184f58487eaf0..9e12d754fe7b5 100644 --- a/buildtools/src/main/resources/log4j2.xml +++ b/buildtools/src/main/resources/log4j2.xml @@ -1,4 +1,4 @@ - + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + diff --git a/buildtools/src/main/resources/pulsar/checkstyle.xml b/buildtools/src/main/resources/pulsar/checkstyle.xml index c63c8993408de..7a32584cc4dd6 100644 --- a/buildtools/src/main/resources/pulsar/checkstyle.xml +++ b/buildtools/src/main/resources/pulsar/checkstyle.xml @@ -1,4 +1,4 @@ - + - - + - - - - - - - - - - - - + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + - - - - + IMPORT CHECKS - - - - + --> + + + + - - - - - + + + + + + + - - - - + + - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + } else --> - else --> - - - - - - - - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + - - - - - + + + diff --git a/buildtools/src/main/resources/pulsar/suppressions.xml b/buildtools/src/main/resources/pulsar/suppressions.xml index 57a01c60f6a27..e558a8a0618fd 100644 --- a/buildtools/src/main/resources/pulsar/suppressions.xml +++ b/buildtools/src/main/resources/pulsar/suppressions.xml @@ -1,4 +1,4 @@ - + - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/filesystem_offload_core_site.xml b/conf/filesystem_offload_core_site.xml index d26cec2cc60f0..3c7e6d94bbe50 100644 --- a/conf/filesystem_offload_core_site.xml +++ b/conf/filesystem_offload_core_site.xml @@ -1,3 +1,4 @@ + - - - fs.defaultFS - - - - hadoop.tmp.dir - pulsar - - - io.file.buffer.size - 4096 - - - io.seqfile.compress.blocksize - 1000000 - - - io.seqfile.compression.type - BLOCK - - - io.map.index.interval - 128 - - + + + fs.defaultFS + + + + hadoop.tmp.dir + pulsar + + + io.file.buffer.size + 4096 + + + io.seqfile.compress.blocksize + 1000000 + + + io.seqfile.compression.type + BLOCK + + + io.map.index.interval + 128 + diff --git a/conf/functions_log4j2.xml b/conf/functions_log4j2.xml index fd4042e82e82f..a1dcc6a1f654a 100644 --- a/conf/functions_log4j2.xml +++ b/conf/functions_log4j2.xml @@ -1,3 +1,4 @@ + - pulsar-functions-instance - 30 - - - pulsar.log.appender - RollingFile - - - pulsar.log.level - info - - - bk.log.level - info - - - - - Console - SYSTEM_OUT - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - RollingFile - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.log - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz - true - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - 1 - true - - - 1 GB - - - 0 0 0 * * ? - - - - - ${sys:pulsar.function.log.dir} - 2 - - ${sys:pulsar.function.log.file}*log.gz - - - 30d - - - - - - BkRollingFile - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz - true - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - 1 - true - - - 1 GB - - - 0 0 0 * * ? - - - - - ${sys:pulsar.function.log.dir} - 2 - - ${sys:pulsar.function.log.file}.bk*log.gz - - - 30d - - - - - - - - org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper - ${sys:bk.log.level} - false - - BkRollingFile - - - - ${sys:pulsar.log.level} - - ${sys:pulsar.log.appender} - ${sys:pulsar.log.level} - - - + pulsar-functions-instance + 30 + + + pulsar.log.appender + RollingFile + + + pulsar.log.level + info + + + bk.log.level + info + + + + + Console + SYSTEM_OUT + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + RollingFile + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.log + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz + true + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + 1 + true + + + 1 GB + + + 0 0 0 * * ? + + + + + ${sys:pulsar.function.log.dir} + 2 + + ${sys:pulsar.function.log.file}*log.gz + + + 30d + + + + + + BkRollingFile + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz + true + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + 1 + true + + + 1 GB + + + 0 0 0 * * ? + + + + + ${sys:pulsar.function.log.dir} + 2 + + ${sys:pulsar.function.log.file}.bk*log.gz + + + 30d + + + + + + + + org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper + ${sys:bk.log.level} + false + + BkRollingFile + + + + ${sys:pulsar.log.level} + + ${sys:pulsar.log.appender} + ${sys:pulsar.log.level} + + + diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml index 4c5b6aab1b7f4..3871c74a88778 100644 --- a/conf/functions_worker.yml +++ b/conf/functions_worker.yml @@ -43,6 +43,16 @@ metadataStoreOperationTimeoutSeconds: 30 # Metadata store cache expiry time in seconds metadataStoreCacheExpirySeconds: 300 +# Specifies if the function worker should use classloading for validating submissions for built-in +# connectors and functions. This is required for validateConnectorConfig to take effect. +# Default is false. +enableClassloadingOfBuiltinFiles: false + +# Specifies if the function worker should use classloading for validating submissions for external +# connectors and functions. This is required for validateConnectorConfig to take effect. +# Default is false. +enableClassloadingOfExternalFiles: false + ################################ # Function package management ################################ @@ -398,9 +408,20 @@ saslJaasServerRoleTokenSignerSecretPath: ######################## connectorsDirectory: ./connectors +# Whether to enable referencing connectors directory files by file url in connector (sink/source) creation +enableReferencingConnectorDirectoryFiles: true +# Regex patterns for enabling creation of connectors by referencing packages in matching http/https urls +additionalEnabledConnectorUrlPatterns: [] functionsDirectory: ./functions - -# Should connector config be validated during submission +# Whether to enable referencing functions directory files by file url in functions creation +enableReferencingFunctionsDirectoryFiles: true +# Regex patterns for enabling creation of functions by referencing packages in matching http/https urls +additionalEnabledFunctionsUrlPatterns: [] + +# Enables extended validation for connector config with fine-grain annotation based validation +# during submission. Classloading with either enableClassloadingOfExternalFiles or +# enableClassloadingOfBuiltinFiles must be enabled on the worker for this to take effect. +# Default is false. validateConnectorConfig: false # Whether to initialize distributed log metadata by runtime. diff --git a/conf/proxy.conf b/conf/proxy.conf index 4194bf7621985..8285e1cb75320 100644 --- a/conf/proxy.conf +++ b/conf/proxy.conf @@ -28,17 +28,19 @@ metadataStoreUrl= # The metadata store URL for the configuration data. If empty, we fall back to use metadataStoreUrl configurationMetadataStoreUrl= -# If Service Discovery is Disabled this url should point to the discovery service provider. +# If does not set metadataStoreUrl or configurationMetadataStoreUrl, this url should point to the discovery service +# provider, and does not support multi urls yet. # The URL must begin with pulsar:// for plaintext or with pulsar+ssl:// for TLS. brokerServiceURL= brokerServiceURLTLS= -# These settings are unnecessary if `zookeeperServers` is specified +# If does not set metadataStoreUrl or configurationMetadataStoreUrl, this url should point to the discovery service +# provider, and does not support multi urls yet. brokerWebServiceURL= brokerWebServiceURLTLS= -# If function workers are setup in a separate cluster, configure the following 2 settings -# to point to the function workers cluster +# If function workers are setup in a separate cluster, configure the following 2 settings. This url should point to +# the discovery service provider of the function workers cluster, and does not support multi urls yet. functionWorkerWebServiceURL= functionWorkerWebServiceURLTLS= diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index df97efcca18a4..bf238fb43920e 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative distribution 3.2.0-SNAPSHOT .. - pulsar-io-distribution pom Pulsar :: Distribution :: IO - ${project.groupId} @@ -46,7 +43,6 @@ provided - @@ -69,9 +65,32 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + - generator-connector-config @@ -151,5 +170,4 @@ - diff --git a/distribution/io/src/assemble/io.xml b/distribution/io/src/assemble/io.xml index 5b652170fdbb5..71d5aaf69f76a 100644 --- a/distribution/io/src/assemble/io.xml +++ b/distribution/io/src/assemble/io.xml @@ -1,3 +1,4 @@ + - + bin dir @@ -38,48 +37,120 @@ . 644 - - - ${basedir}/../../pulsar-io/cassandra/target/pulsar-io-cassandra-${project.version}.nar - ${basedir}/../../pulsar-io/twitter/target/pulsar-io-twitter-${project.version}.nar - ${basedir}/../../pulsar-io/kafka/target/pulsar-io-kafka-${project.version}.nar - ${basedir}/../../pulsar-io/http/target/pulsar-io-http-${project.version}.nar - ${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar - ${basedir}/../../pulsar-io/rabbitmq/target/pulsar-io-rabbitmq-${project.version}.nar - ${basedir}/../../pulsar-io/nsq/target/pulsar-io-nsq-${project.version}.nar - ${basedir}/../../pulsar-io/jdbc/sqlite/target/pulsar-io-jdbc-sqlite-${project.version}.nar - ${basedir}/../../pulsar-io/jdbc/mariadb/target/pulsar-io-jdbc-mariadb-${project.version}.nar - ${basedir}/../../pulsar-io/jdbc/clickhouse/target/pulsar-io-jdbc-clickhouse-${project.version}.nar - ${basedir}/../../pulsar-io/jdbc/postgres/target/pulsar-io-jdbc-postgres-${project.version}.nar - ${basedir}/../../pulsar-io/jdbc/openmldb/target/pulsar-io-jdbc-openmldb-${project.version}.nar - ${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar - ${basedir}/../../pulsar-io/batch-data-generator/target/pulsar-io-batch-data-generator-${project.version}.nar - ${basedir}/../../pulsar-io/aerospike/target/pulsar-io-aerospike-${project.version}.nar - ${basedir}/../../pulsar-io/elastic-search/target/pulsar-io-elastic-search-${project.version}.nar - ${basedir}/../../pulsar-io/kafka-connect-adaptor-nar/target/pulsar-io-kafka-connect-adaptor-${project.version}.nar - ${basedir}/../../pulsar-io/hbase/target/pulsar-io-hbase-${project.version}.nar - ${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar - ${basedir}/../../pulsar-io/hdfs2/target/pulsar-io-hdfs2-${project.version}.nar - ${basedir}/../../pulsar-io/hdfs3/target/pulsar-io-hdfs3-${project.version}.nar - ${basedir}/../../pulsar-io/file/target/pulsar-io-file-${project.version}.nar - ${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar - ${basedir}/../../pulsar-io/canal/target/pulsar-io-canal-${project.version}.nar - ${basedir}/../../pulsar-io/netty/target/pulsar-io-netty-${project.version}.nar - ${basedir}/../../pulsar-io/mongo/target/pulsar-io-mongo-${project.version}.nar - ${basedir}/../../pulsar-io/debezium/mysql/target/pulsar-io-debezium-mysql-${project.version}.nar - ${basedir}/../../pulsar-io/debezium/postgres/target/pulsar-io-debezium-postgres-${project.version}.nar - ${basedir}/../../pulsar-io/debezium/oracle/target/pulsar-io-debezium-oracle-${project.version}.nar - ${basedir}/../../pulsar-io/debezium/mssql/target/pulsar-io-debezium-mssql-${project.version}.nar - ${basedir}/../../pulsar-io/debezium/mongodb/target/pulsar-io-debezium-mongodb-${project.version}.nar - ${basedir}/../../pulsar-io/influxdb/target/pulsar-io-influxdb-${project.version}.nar - ${basedir}/../../pulsar-io/redis/target/pulsar-io-redis-${project.version}.nar - ${basedir}/../../pulsar-io/flume/target/pulsar-io-flume-${project.version}.nar - ${basedir}/../../pulsar-io/solr/target/pulsar-io-solr-${project.version}.nar - ${basedir}/../../pulsar-io/dynamodb/target/pulsar-io-dynamodb-${project.version}.nar - ${basedir}/../../pulsar-io/alluxio/target/pulsar-io-alluxio-${project.version}.nar + + ${basedir}/../../pulsar-io/cassandra/target/pulsar-io-cassandra-${project.version}.nar + + + ${basedir}/../../pulsar-io/twitter/target/pulsar-io-twitter-${project.version}.nar + + + ${basedir}/../../pulsar-io/kafka/target/pulsar-io-kafka-${project.version}.nar + + + ${basedir}/../../pulsar-io/http/target/pulsar-io-http-${project.version}.nar + + + ${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar + + + ${basedir}/../../pulsar-io/rabbitmq/target/pulsar-io-rabbitmq-${project.version}.nar + + + ${basedir}/../../pulsar-io/nsq/target/pulsar-io-nsq-${project.version}.nar + + + ${basedir}/../../pulsar-io/jdbc/sqlite/target/pulsar-io-jdbc-sqlite-${project.version}.nar + + + ${basedir}/../../pulsar-io/jdbc/mariadb/target/pulsar-io-jdbc-mariadb-${project.version}.nar + + + ${basedir}/../../pulsar-io/jdbc/clickhouse/target/pulsar-io-jdbc-clickhouse-${project.version}.nar + + + ${basedir}/../../pulsar-io/jdbc/postgres/target/pulsar-io-jdbc-postgres-${project.version}.nar + + + ${basedir}/../../pulsar-io/jdbc/openmldb/target/pulsar-io-jdbc-openmldb-${project.version}.nar + + + ${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar + + + ${basedir}/../../pulsar-io/batch-data-generator/target/pulsar-io-batch-data-generator-${project.version}.nar + + + ${basedir}/../../pulsar-io/aerospike/target/pulsar-io-aerospike-${project.version}.nar + + + ${basedir}/../../pulsar-io/elastic-search/target/pulsar-io-elastic-search-${project.version}.nar + + + ${basedir}/../../pulsar-io/kafka-connect-adaptor-nar/target/pulsar-io-kafka-connect-adaptor-${project.version}.nar + + + ${basedir}/../../pulsar-io/hbase/target/pulsar-io-hbase-${project.version}.nar + + + ${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar + + + ${basedir}/../../pulsar-io/hdfs2/target/pulsar-io-hdfs2-${project.version}.nar + + + ${basedir}/../../pulsar-io/hdfs3/target/pulsar-io-hdfs3-${project.version}.nar + + + ${basedir}/../../pulsar-io/file/target/pulsar-io-file-${project.version}.nar + + + ${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar + + + ${basedir}/../../pulsar-io/canal/target/pulsar-io-canal-${project.version}.nar + + + ${basedir}/../../pulsar-io/netty/target/pulsar-io-netty-${project.version}.nar + + + ${basedir}/../../pulsar-io/mongo/target/pulsar-io-mongo-${project.version}.nar + + + ${basedir}/../../pulsar-io/debezium/mysql/target/pulsar-io-debezium-mysql-${project.version}.nar + + + ${basedir}/../../pulsar-io/debezium/postgres/target/pulsar-io-debezium-postgres-${project.version}.nar + + + ${basedir}/../../pulsar-io/debezium/oracle/target/pulsar-io-debezium-oracle-${project.version}.nar + + + ${basedir}/../../pulsar-io/debezium/mssql/target/pulsar-io-debezium-mssql-${project.version}.nar + + + ${basedir}/../../pulsar-io/debezium/mongodb/target/pulsar-io-debezium-mongodb-${project.version}.nar + + + ${basedir}/../../pulsar-io/influxdb/target/pulsar-io-influxdb-${project.version}.nar + + + ${basedir}/../../pulsar-io/redis/target/pulsar-io-redis-${project.version}.nar + + + ${basedir}/../../pulsar-io/flume/target/pulsar-io-flume-${project.version}.nar + + + ${basedir}/../../pulsar-io/solr/target/pulsar-io-solr-${project.version}.nar + + + ${basedir}/../../pulsar-io/dynamodb/target/pulsar-io-dynamodb-${project.version}.nar + + + ${basedir}/../../pulsar-io/alluxio/target/pulsar-io-alluxio-${project.version}.nar + diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index 0fd0d1dc2bc79..fba25b4b846b4 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative distribution 3.2.0-SNAPSHOT .. - pulsar-offloader-distribution pom Pulsar :: Distribution :: Offloader - ${project.groupId} @@ -66,7 +63,6 @@ - @@ -90,6 +86,30 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/distribution/offloaders/src/assemble/offloaders.xml b/distribution/offloaders/src/assemble/offloaders.xml index 38f7eee906064..19c60f68c21d4 100644 --- a/distribution/offloaders/src/assemble/offloaders.xml +++ b/distribution/offloaders/src/assemble/offloaders.xml @@ -1,3 +1,4 @@ + - + bin tar.gz @@ -38,13 +37,11 @@ . 644 - ${basedir}/../../tiered-storage/jcloud/target/tiered-storage-jcloud-${project.version}.nar offloaders 644 - ${basedir}/../../tiered-storage/file-system/target/tiered-storage-file-system-${project.version}.nar offloaders diff --git a/distribution/pom.xml b/distribution/pom.xml index 3f52a30bb5237..2023cd71de6a8 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - distribution pom Pulsar :: Distribution - - main @@ -47,7 +43,6 @@ shell - core-modules @@ -55,7 +50,6 @@ - @@ -65,6 +59,30 @@ ${skipBuildDistribution} + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 647de84bf07f3..34e111c7f9c48 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative distribution 3.2.0-SNAPSHOT .. - pulsar-server-distribution pom Pulsar :: Distribution :: Server - ${project.groupId} pulsar-broker ${project.version} - ${project.groupId} pulsar-docs-tools ${project.version} - ${project.groupId} pulsar-proxy ${project.version} - ${project.groupId} pulsar-broker-auth-oidc ${project.version} - ${project.groupId} pulsar-broker-auth-sasl ${project.version} - ${project.groupId} pulsar-client-auth-sasl ${project.version} - jline jline ${jline.version} - org.apache.zookeeper zookeeper-prometheus-metrics @@ -95,7 +85,6 @@ - ${project.groupId} pulsar-package-bookkeeper-storage @@ -107,13 +96,11 @@ - ${project.groupId} pulsar-package-filesystem-storage ${project.version} - ${project.groupId} pulsar-client-tools @@ -125,33 +112,27 @@ - ${project.groupId} pulsar-testclient ${project.version} - org.apache.logging.log4j log4j-api - org.apache.logging.log4j log4j-core - org.apache.logging.log4j log4j-web - io.dropwizard.metrics metrics-core - io.dropwizard.metrics metrics-graphite @@ -162,44 +143,36 @@ - io.dropwizard.metrics metrics-jvm - org.xerial.snappy snappy-java - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - org.apache.logging.log4j log4j-slf4j-impl - org.apache.bookkeeper.stats prometheus-metrics-provider - io.prometheus simpleclient_log4j2 - ${project.groupId} bouncy-castle-bc ${project.version} pkg - ${project.groupId} @@ -214,7 +187,6 @@ - ${project.groupId} pulsar-functions-worker @@ -231,7 +203,6 @@ - ${project.groupId} @@ -246,7 +217,6 @@ - ${project.groupId} @@ -259,7 +229,6 @@ - io.grpc @@ -269,13 +238,11 @@ org.bouncycastle bcpkix-jdk18on - io.perfmark perfmark-api compile - org.apache.bookkeeper.http @@ -297,7 +264,6 @@ vertx-web - @@ -338,6 +304,30 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index db66fc4ced547..b3cecd215f3ce 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -285,7 +285,7 @@ The Apache Software License, Version 2.0 - commons-lang-commons-lang-2.6.jar - commons-logging-commons-logging-1.1.1.jar - org.apache.commons-commons-collections4-4.4.jar - - org.apache.commons-commons-compress-1.21.jar + - org.apache.commons-commons-compress-1.26.0.jar - org.apache.commons-commons-lang3-3.11.jar - org.apache.commons-commons-text-1.10.0.jar * Netty @@ -345,34 +345,34 @@ The Apache Software License, Version 2.0 - net.java.dev.jna-jna-jpms-5.12.1.jar - net.java.dev.jna-jna-platform-jpms-5.12.1.jar * BookKeeper - - org.apache.bookkeeper-bookkeeper-common-4.16.3.jar - - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.3.jar - - org.apache.bookkeeper-bookkeeper-proto-4.16.3.jar - - org.apache.bookkeeper-bookkeeper-server-4.16.3.jar - - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.3.jar - - org.apache.bookkeeper-circe-checksum-4.16.3.jar - - org.apache.bookkeeper-cpu-affinity-4.16.3.jar - - org.apache.bookkeeper-statelib-4.16.3.jar - - org.apache.bookkeeper-stream-storage-api-4.16.3.jar - - org.apache.bookkeeper-stream-storage-common-4.16.3.jar - - org.apache.bookkeeper-stream-storage-java-client-4.16.3.jar - - org.apache.bookkeeper-stream-storage-java-client-base-4.16.3.jar - - org.apache.bookkeeper-stream-storage-proto-4.16.3.jar - - org.apache.bookkeeper-stream-storage-server-4.16.3.jar - - org.apache.bookkeeper-stream-storage-service-api-4.16.3.jar - - org.apache.bookkeeper-stream-storage-service-impl-4.16.3.jar - - org.apache.bookkeeper.http-http-server-4.16.3.jar - - org.apache.bookkeeper.http-vertx-http-server-4.16.3.jar - - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.3.jar - - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.3.jar - - org.apache.distributedlog-distributedlog-common-4.16.3.jar - - org.apache.distributedlog-distributedlog-core-4.16.3-tests.jar - - org.apache.distributedlog-distributedlog-core-4.16.3.jar - - org.apache.distributedlog-distributedlog-protocol-4.16.3.jar - - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.3.jar - - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.3.jar - - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.3.jar - - org.apache.bookkeeper-native-io-4.16.3.jar + - org.apache.bookkeeper-bookkeeper-common-4.16.4.jar + - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.4.jar + - org.apache.bookkeeper-bookkeeper-proto-4.16.4.jar + - org.apache.bookkeeper-bookkeeper-server-4.16.4.jar + - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.4.jar + - org.apache.bookkeeper-circe-checksum-4.16.4.jar + - org.apache.bookkeeper-cpu-affinity-4.16.4.jar + - org.apache.bookkeeper-statelib-4.16.4.jar + - org.apache.bookkeeper-stream-storage-api-4.16.4.jar + - org.apache.bookkeeper-stream-storage-common-4.16.4.jar + - org.apache.bookkeeper-stream-storage-java-client-4.16.4.jar + - org.apache.bookkeeper-stream-storage-java-client-base-4.16.4.jar + - org.apache.bookkeeper-stream-storage-proto-4.16.4.jar + - org.apache.bookkeeper-stream-storage-server-4.16.4.jar + - org.apache.bookkeeper-stream-storage-service-api-4.16.4.jar + - org.apache.bookkeeper-stream-storage-service-impl-4.16.4.jar + - org.apache.bookkeeper.http-http-server-4.16.4.jar + - org.apache.bookkeeper.http-vertx-http-server-4.16.4.jar + - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.4.jar + - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.4.jar + - org.apache.distributedlog-distributedlog-common-4.16.4.jar + - org.apache.distributedlog-distributedlog-core-4.16.4-tests.jar + - org.apache.distributedlog-distributedlog-core-4.16.4.jar + - org.apache.distributedlog-distributedlog-protocol-4.16.4.jar + - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.4.jar + - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.4.jar + - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.4.jar + - org.apache.bookkeeper-native-io-4.16.4.jar * Apache HTTP Client - org.apache.httpcomponents-httpclient-4.5.13.jar - org.apache.httpcomponents-httpcore-4.4.15.jar @@ -382,25 +382,25 @@ The Apache Software License, Version 2.0 - org.asynchttpclient-async-http-client-2.12.1.jar - org.asynchttpclient-async-http-client-netty-utils-2.12.1.jar * Jetty - - org.eclipse.jetty-jetty-client-9.4.53.v20231009.jar - - org.eclipse.jetty-jetty-continuation-9.4.53.v20231009.jar - - org.eclipse.jetty-jetty-http-9.4.53.v20231009.jar - - org.eclipse.jetty-jetty-io-9.4.53.v20231009.jar - - org.eclipse.jetty-jetty-proxy-9.4.53.v20231009.jar - - org.eclipse.jetty-jetty-security-9.4.53.v20231009.jar - - org.eclipse.jetty-jetty-server-9.4.53.v20231009.jar - - org.eclipse.jetty-jetty-servlet-9.4.53.v20231009.jar - - org.eclipse.jetty-jetty-servlets-9.4.53.v20231009.jar - - org.eclipse.jetty-jetty-util-9.4.53.v20231009.jar - - org.eclipse.jetty-jetty-util-ajax-9.4.53.v20231009.jar - - org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.53.v20231009.jar - - org.eclipse.jetty.websocket-websocket-api-9.4.53.v20231009.jar - - org.eclipse.jetty.websocket-websocket-client-9.4.53.v20231009.jar - - org.eclipse.jetty.websocket-websocket-common-9.4.53.v20231009.jar - - org.eclipse.jetty.websocket-websocket-server-9.4.53.v20231009.jar - - org.eclipse.jetty.websocket-websocket-servlet-9.4.53.v20231009.jar - - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.53.v20231009.jar - - org.eclipse.jetty-jetty-alpn-server-9.4.53.v20231009.jar + - org.eclipse.jetty-jetty-client-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-continuation-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-http-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-io-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-proxy-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-security-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-server-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-servlet-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-servlets-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-util-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-util-ajax-9.4.54.v20240208.jar + - org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.54.v20240208.jar + - org.eclipse.jetty.websocket-websocket-api-9.4.54.v20240208.jar + - org.eclipse.jetty.websocket-websocket-client-9.4.54.v20240208.jar + - org.eclipse.jetty.websocket-websocket-common-9.4.54.v20240208.jar + - org.eclipse.jetty.websocket-websocket-server-9.4.54.v20240208.jar + - org.eclipse.jetty.websocket-websocket-servlet-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-alpn-server-9.4.54.v20240208.jar * SnakeYaml -- org.yaml-snakeyaml-2.0.jar * RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.5.1.jar @@ -445,6 +445,10 @@ The Apache Software License, Version 2.0 * Jodah - net.jodah-typetools-0.5.0.jar - net.jodah-failsafe-2.4.4.jar + * Byte Buddy + - net.bytebuddy-byte-buddy-1.14.12.jar + * zt-zip + - org.zeroturnaround-zt-zip-1.17.jar * Apache Avro - org.apache.avro-avro-1.11.3.jar - org.apache.avro-avro-protobuf-1.11.3.jar diff --git a/distribution/server/src/assemble/bin.xml b/distribution/server/src/assemble/bin.xml index 949c265706929..31b17eacda1b3 100644 --- a/distribution/server/src/assemble/bin.xml +++ b/distribution/server/src/assemble/bin.xml @@ -1,3 +1,4 @@ + - + bin tar.gz @@ -115,14 +114,11 @@ ${artifact.groupId}-${artifact.artifactId}-${artifact.version}${dashClassifier?}.${artifact.extension} - - org.apache.pulsar:pulsar-functions-runtime-all - + io.streamnative:pulsar-functions-runtime-all org.projectlombok:lombok - - org.apache.pulsar:pulsar-functions-api-examples + io.streamnative:pulsar-functions-api-examples *:tar.gz diff --git a/distribution/shell/pom.xml b/distribution/shell/pom.xml index a1146811ff23f..4f5cf8f97c3f5 100644 --- a/distribution/shell/pom.xml +++ b/distribution/shell/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative distribution 3.2.0-SNAPSHOT .. - pulsar-shell-distribution pom Pulsar :: Distribution :: Shell - ${project.groupId} pulsar-client-tools ${project.version} - org.apache.logging.log4j log4j-core - org.apache.logging.log4j log4j-web @@ -53,14 +48,11 @@ org.apache.logging.log4j log4j-slf4j-impl - io.prometheus simpleclient_log4j2 - - @@ -101,6 +93,30 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index bdd1b18ce0c3d..8047501b15661 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -342,7 +342,7 @@ The Apache Software License, Version 2.0 - commons-logging-1.2.jar - commons-lang3-3.11.jar - commons-text-1.10.0.jar - - commons-compress-1.21.jar + - commons-compress-1.26.0.jar * Netty - netty-buffer-4.1.104.Final.jar - netty-codec-4.1.104.Final.jar @@ -386,23 +386,23 @@ The Apache Software License, Version 2.0 - log4j-web-2.18.0.jar * BookKeeper - - bookkeeper-common-allocator-4.16.3.jar - - cpu-affinity-4.16.3.jar - - circe-checksum-4.16.3.jar + - bookkeeper-common-allocator-4.16.4.jar + - cpu-affinity-4.16.4.jar + - circe-checksum-4.16.4.jar * AirCompressor - aircompressor-0.20.jar * AsyncHttpClient - async-http-client-2.12.1.jar - async-http-client-netty-utils-2.12.1.jar * Jetty - - jetty-client-9.4.53.v20231009.jar - - jetty-http-9.4.53.v20231009.jar - - jetty-io-9.4.53.v20231009.jar - - jetty-util-9.4.53.v20231009.jar - - javax-websocket-client-impl-9.4.53.v20231009.jar - - websocket-api-9.4.53.v20231009.jar - - websocket-client-9.4.53.v20231009.jar - - websocket-common-9.4.53.v20231009.jar + - jetty-client-9.4.54.v20240208.jar + - jetty-http-9.4.54.v20240208.jar + - jetty-io-9.4.54.v20240208.jar + - jetty-util-9.4.54.v20240208.jar + - javax-websocket-client-impl-9.4.54.v20240208.jar + - websocket-api-9.4.54.v20240208.jar + - websocket-client-9.4.54.v20240208.jar + - websocket-common-9.4.54.v20240208.jar * SnakeYaml -- snakeyaml-2.0.jar * Google Error Prone Annotations - error_prone_annotations-2.5.1.jar * Javassist -- javassist-3.25.0-GA.jar diff --git a/distribution/shell/src/assemble/shell.xml b/distribution/shell/src/assemble/shell.xml index f823e0258b231..cd11bd8a1ef57 100644 --- a/distribution/shell/src/assemble/shell.xml +++ b/distribution/shell/src/assemble/shell.xml @@ -1,3 +1,4 @@ + - + bin tar.gz @@ -46,7 +45,6 @@ . 644 - bin ${basedir}/../../bin/pulsar-admin-common.sh @@ -76,7 +74,6 @@ ${basedir}/../../conf/log4j2.yaml - lib diff --git a/docker/pom.xml b/docker/pom.xml index 0df3c066babd4..779b58e642000 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -1,4 +1,4 @@ - + - + pom 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT @@ -39,6 +38,30 @@ ${skipBuildDistribution} + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index 269a2e4f98f62..6b4765327e319 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar + io.streamnative docker-images 3.2.0-SNAPSHOT @@ -29,7 +29,6 @@ pulsar-all-docker-image Apache Pulsar :: Docker Images :: Pulsar Latest Version (Include All Components) pom - ${project.groupId} @@ -64,7 +63,6 @@ - git-commit-id-no-git @@ -177,7 +175,6 @@ - docker-push @@ -200,6 +197,5 @@ - diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 4e5885ce55d17..6a0dc0100e7fd 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -36,10 +36,14 @@ COPY scripts/install-pulsar-client.sh /pulsar/bin # The final image needs to give the root group sufficient permission for Pulsar components # to write to specific directories within /pulsar +# The ownership is changed to uid 10000 to allow using a different root group. This is necessary when running the +# container when gid=0 is prohibited. In that case, the container must be run with uid 10000 with +# any group id != 0 (for example 10001). # The file permissions are preserved when copying files from this builder image to the target image. RUN for SUBDIRECTORY in conf data download logs; do \ [ -d /pulsar/$SUBDIRECTORY ] || mkdir /pulsar/$SUBDIRECTORY; \ - chmod -R g+w /pulsar/$SUBDIRECTORY; \ + chmod -R ug+w /pulsar/$SUBDIRECTORY; \ + chown -R 10000:0 /pulsar/$SUBDIRECTORY; \ done ### Create 2nd stage from Ubuntu image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 1c29fa3f00c9c..743e649c880aa 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar + io.streamnative docker-images 3.2.0-SNAPSHOT @@ -29,7 +29,6 @@ pulsar-docker-image Apache Pulsar :: Docker Images :: Pulsar Latest Version pom - ${project.groupId} @@ -46,13 +45,11 @@ - http://archive.ubuntu.com/ubuntu/ http://security.ubuntu.com/ubuntu/ 17 - git-commit-id-no-git @@ -130,7 +127,6 @@ - docker-push @@ -153,6 +149,5 @@ - diff --git a/docker/pulsar/scripts/apply-config-from-env-with-prefix.py b/docker/pulsar/scripts/apply-config-from-env-with-prefix.py index 58f6c98975005..9943b283a9f89 100755 --- a/docker/pulsar/scripts/apply-config-from-env-with-prefix.py +++ b/docker/pulsar/scripts/apply-config-from-env-with-prefix.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -32,83 +32,8 @@ # update if they exist and ignored if they don't. ############################################################ -import os -import sys - -if len(sys.argv) < 3: - print('Usage: %s [...]' % (sys.argv[0])) - sys.exit(1) - -# Always apply env config to env scripts as well -prefix = sys.argv[1] -conf_files = sys.argv[2:] - -PF_ENV_DEBUG = (os.environ.get('PF_ENV_DEBUG','0') == '1') - -for conf_filename in conf_files: - lines = [] # List of config file lines - keys = {} # Map a key to its line number in the file - - # Load conf file - for line in open(conf_filename): - lines.append(line) - line = line.strip() - if not line or line.startswith('#'): - continue - - try: - k,v = line.split('=', 1) - keys[k] = len(lines) - 1 - except: - if PF_ENV_DEBUG: - print("[%s] skip Processing %s" % (conf_filename, line)) - - # Update values from Env - for k in sorted(os.environ.keys()): - v = os.environ[k].strip() - - # Hide the value in logs if is password. - if "password" in k.lower(): - displayValue = "********" - else: - displayValue = v - - if k.startswith(prefix): - k = k[len(prefix):] - if k in keys: - print('[%s] Applying config %s = %s' % (conf_filename, k, displayValue)) - idx = keys[k] - lines[idx] = '%s=%s\n' % (k, v) - - - # Ensure we have a new-line at the end of the file, to avoid issue - # when appending more lines to the config - lines.append('\n') - - # Add new keys from Env - for k in sorted(os.environ.keys()): - v = os.environ[k] - if not k.startswith(prefix): - continue - - # Hide the value in logs if is password. - if "password" in k.lower(): - displayValue = "********" - else: - displayValue = v - - k = k[len(prefix):] - if k not in keys: - print('[%s] Adding config %s = %s' % (conf_filename, k, displayValue)) - lines.append('%s=%s\n' % (k, v)) - else: - print('[%s] Updating config %s = %s' % (conf_filename, k, displayValue)) - lines[keys[k]] = '%s=%s\n' % (k, v) - - - # Store back the updated config in the same file - f = open(conf_filename, 'w') - for line in lines: - f.write(line) - f.close() +# DEPRECATED: Use "apply-config-from-env.py --prefix MY_PREFIX_ conf_file" instead +# this is not a python script, but a bash script. Call apply-config-from-env.py with the prefix argument +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +"${SCRIPT_DIR}/apply-config-from-env.py" --prefix "$1" "${@:2}" diff --git a/docker/pulsar/scripts/apply-config-from-env.py b/docker/pulsar/scripts/apply-config-from-env.py index b8b479fc15b85..da51f05f8be66 100755 --- a/docker/pulsar/scripts/apply-config-from-env.py +++ b/docker/pulsar/scripts/apply-config-from-env.py @@ -25,18 +25,29 @@ ## ./apply-config-from-env file.conf ## -import os, sys - -if len(sys.argv) < 2: - print('Usage: %s' % (sys.argv[0])) +import os, sys, argparse + +parser = argparse.ArgumentParser(description='Pulsar configuration file customizer based on environment variables') +parser.add_argument('--prefix', default='PULSAR_PREFIX_', help='Prefix for environment variables, default is PULSAR_PREFIX_') +parser.add_argument('conf_files', nargs='*', help='Configuration files') +args = parser.parse_args() +if not args.conf_files: + parser.print_help() sys.exit(1) -# Always apply env config to env scripts as well -conf_files = sys.argv[1:] +env_prefix = args.prefix +conf_files = args.conf_files -PF_ENV_PREFIX = 'PULSAR_PREFIX_' PF_ENV_DEBUG = (os.environ.get('PF_ENV_DEBUG','0') == '1') +# List of keys where the value should not be displayed in logs +sensitive_keys = ["brokerClientAuthenticationParameters", "bookkeeperClientAuthenticationParameters", "tokenSecretKey"] + +def sanitize_display_value(k, v): + if "password" in k.lower() or k in sensitive_keys or (k == "tokenSecretKey" and v.startswith("data:")): + return "********" + return v + for conf_filename in conf_files: lines = [] # List of config file lines keys = {} # Map a key to its line number in the file @@ -47,7 +58,6 @@ line = line.strip() if not line: continue - try: k,v = line.split('=', 1) if k.startswith('#'): @@ -61,37 +71,26 @@ for k in sorted(os.environ.keys()): v = os.environ[k].strip() - # Hide the value in logs if is password. - if "password" in k.lower(): - displayValue = "********" - else: - displayValue = v - - if k.startswith(PF_ENV_PREFIX): - k = k[len(PF_ENV_PREFIX):] if k in keys: + displayValue = sanitize_display_value(k, v) print('[%s] Applying config %s = %s' % (conf_filename, k, displayValue)) idx = keys[k] lines[idx] = '%s=%s\n' % (k, v) - # Ensure we have a new-line at the end of the file, to avoid issue # when appending more lines to the config lines.append('\n') - - # Add new keys from Env + + # Add new keys from Env for k in sorted(os.environ.keys()): - v = os.environ[k] - if not k.startswith(PF_ENV_PREFIX): + if not k.startswith(env_prefix): continue - # Hide the value in logs if is password. - if "password" in k.lower(): - displayValue = "********" - else: - displayValue = v + v = os.environ[k].strip() + k = k[len(env_prefix):] + + displayValue = sanitize_display_value(k, v) - k = k[len(PF_ENV_PREFIX):] if k not in keys: print('[%s] Adding config %s = %s' % (conf_filename, k, displayValue)) lines.append('%s=%s\n' % (k, v)) @@ -99,10 +98,8 @@ print('[%s] Updating config %s = %s' % (conf_filename, k, displayValue)) lines[keys[k]] = '%s=%s\n' % (k, v) - # Store back the updated config in the same file f = open(conf_filename, 'w') for line in lines: f.write(line) - f.close() - + f.close() \ No newline at end of file diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index 19bbe37d99dbb..e7934969b32ee 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - jclouds-shaded Apache Pulsar :: Jclouds shaded - - org.apache.jclouds jclouds-allblobstore @@ -66,7 +61,6 @@ javax.annotation-api - @@ -82,7 +76,6 @@ true true false - com.google.guava:guava @@ -106,7 +99,6 @@ com.google.errorprone:* - com.google @@ -140,21 +132,20 @@ com.google.errorprone org.apache.pulsar.jcloud.shade.com.google.errorprone - - - org.apache.jclouds:jclouds-core - - - lib/gson*jar - - - + + org.apache.jclouds:jclouds-core + + + lib/gson*jar + + + diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 318a5377d14fa..f0cadc45ecdd1 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - managed-ledger Managed Ledger - org.apache.bookkeeper bookkeeper-server - org.apache.bookkeeper.stats prometheus-metrics-provider - org.apache.bookkeeper.stats codahale-metrics-provider @@ -54,36 +49,30 @@ - com.google.protobuf protobuf-java - ${project.groupId} pulsar-common ${project.version} - ${project.groupId} pulsar-metadata ${project.version} - com.google.guava guava - ${project.groupId} testmocks ${project.version} test - org.apache.zookeeper zookeeper @@ -105,29 +94,25 @@ - io.dropwizard.metrics - metrics-core - test + io.dropwizard.metrics + metrics-core + test - org.xerial.snappy - snappy-java - test + org.xerial.snappy + snappy-java + test - org.awaitility awaitility test - org.slf4j slf4j-api - - @@ -147,7 +132,6 @@ - org.apache.maven.plugins maven-jar-plugin diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java index d1ffdf6d2d763..bc6a1e9a782d6 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java @@ -517,6 +517,10 @@ void markDelete(Position position, Map properties) */ void rewind(); + default void rewind(boolean readCompacted) { + rewind(); + } + /** * Move the cursor to a different read position. * diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java index 0c93a5b642cf6..6ee9c2f949243 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java @@ -170,6 +170,7 @@ public int getMinimumRolloverTimeMs() { * the time unit */ public void setMinimumRolloverTime(int minimumRolloverTime, TimeUnit unit) { + checkArgument(minimumRolloverTime >= 0); this.minimumRolloverTimeMs = (int) unit.toMillis(minimumRolloverTime); checkArgument(maximumRolloverTimeMs >= minimumRolloverTimeMs, "Minimum rollover time needs to be less than maximum rollover time"); @@ -195,6 +196,7 @@ public long getMaximumRolloverTimeMs() { * the time unit */ public void setMaximumRolloverTime(int maximumRolloverTime, TimeUnit unit) { + checkArgument(maximumRolloverTime >= 0); this.maximumRolloverTimeMs = unit.toMillis(maximumRolloverTime); checkArgument(maximumRolloverTimeMs >= minimumRolloverTimeMs, "Maximum rollover time needs to be greater than minimum rollover time"); @@ -411,7 +413,8 @@ public ManagedLedgerConfig setThrottleMarkDelete(double throttleMarkDelete) { * time unit for retention time */ public ManagedLedgerConfig setRetentionTime(int retentionTime, TimeUnit unit) { - this.retentionTimeMs = unit.toMillis(retentionTime); + checkArgument(retentionTime >= -1, "The retention time should be -1, 0 or value > 0"); + this.retentionTimeMs = retentionTime != -1 ? unit.toMillis(retentionTime) : -1; return this; } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 4b65d62f0eee8..da013c0731307 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -59,6 +59,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Function; import java.util.function.Predicate; +import java.util.stream.LongStream; import org.apache.bookkeeper.client.AsyncCallback.CloseCallback; import org.apache.bookkeeper.client.AsyncCallback.OpenCallback; import org.apache.bookkeeper.client.BKException; @@ -195,11 +196,11 @@ public class ManagedCursorImpl implements ManagedCursor { position.ackSet = null; return position; }; - private final RangeSetWrapper individualDeletedMessages; + protected final RangeSetWrapper individualDeletedMessages; // Maintain the deletion status for batch messages // (ledgerId, entryId) -> deletion indexes - private final ConcurrentSkipListMap batchDeletedIndexes; + protected final ConcurrentSkipListMap batchDeletedIndexes; private final ReadWriteLock lock = new ReentrantReadWriteLock(); private RateLimiter markDeleteLimiter; @@ -676,7 +677,7 @@ private void recoveredCursor(PositionImpl position, Map properties LedgerHandle recoveredFromCursorLedger) { // if the position was at a ledger that didn't exist (since it will be deleted if it was previously empty), // we need to move to the next existing ledger - if (!ledger.ledgerExists(position.getLedgerId())) { + if (position.getEntryId() == -1L && !ledger.ledgerExists(position.getLedgerId())) { Long nextExistingLedger = ledger.getNextValidLedger(position.getLedgerId()); if (nextExistingLedger == null) { log.info("[{}] [{}] Couldn't find next next valid ledger for recovery {}", ledger.getName(), name, @@ -786,6 +787,8 @@ public void asyncReadEntriesWithSkip(int numberOfEntriesToRead, long maxSizeByte int numOfEntriesToRead = applyMaxSizeCap(numberOfEntriesToRead, maxSizeBytes); PENDING_READ_OPS_UPDATER.incrementAndGet(this); + // Skip deleted entries. + skipCondition = skipCondition == null ? this::isMessageDeleted : skipCondition.or(this::isMessageDeleted); OpReadEntry op = OpReadEntry.create(this, readPosition, numOfEntriesToRead, callback, ctx, maxPosition, skipCondition); ledger.asyncReadEntries(op); @@ -944,6 +947,8 @@ public void asyncReadEntriesWithSkipOrWait(int maxEntries, long maxSizeBytes, Re asyncReadEntriesWithSkip(numberOfEntriesToRead, NO_MAX_SIZE_LIMIT, callback, ctx, maxPosition, skipCondition); } else { + // Skip deleted entries. + skipCondition = skipCondition == null ? this::isMessageDeleted : skipCondition.or(this::isMessageDeleted); OpReadEntry op = OpReadEntry.create(this, readPosition, numberOfEntriesToRead, callback, ctx, maxPosition, skipCondition); @@ -2512,9 +2517,15 @@ public Position getPersistentMarkDeletedPosition() { @Override public void rewind() { + rewind(false); + } + + @Override + public void rewind(boolean readCompacted) { lock.writeLock().lock(); try { - PositionImpl newReadPosition = ledger.getNextValidPosition(markDeletePosition); + PositionImpl newReadPosition = + readCompacted ? markDeletePosition.getNext() : ledger.getNextValidPosition(markDeletePosition); PositionImpl oldReadPosition = readPosition; log.info("[{}-{}] Rewind from {} to {}", ledger.getName(), name, oldReadPosition, newReadPosition); @@ -2784,30 +2795,23 @@ public void skipNonRecoverableLedger(final long ledgerId){ if (ledgerInfo == null) { return; } - lock.writeLock().lock(); log.warn("[{}] [{}] Since the ledger [{}] is lost and the autoSkipNonRecoverableData is true, this ledger will" + " be auto acknowledge in subscription", ledger.getName(), name, ledgerId); - try { - for (int i = 0; i < ledgerInfo.getEntries(); i++) { - if (!individualDeletedMessages.contains(ledgerId, i)) { - asyncDelete(PositionImpl.get(ledgerId, i), new AsyncCallbacks.DeleteCallback() { - @Override - public void deleteComplete(Object ctx) { - // ignore. - } + asyncDelete(() -> LongStream.range(0, ledgerInfo.getEntries()) + .mapToObj(i -> (Position) PositionImpl.get(ledgerId, i)).iterator(), + new AsyncCallbacks.DeleteCallback() { + @Override + public void deleteComplete(Object ctx) { + // ignore. + } - @Override - public void deleteFailed(ManagedLedgerException ex, Object ctx) { - // The method internalMarkDelete already handled the failure operation. We only need to - // make sure the memory state is updated. - // If the broker crashed, the non-recoverable ledger will be detected again. - } - }, null); - } - } - } finally { - lock.writeLock().unlock(); - } + @Override + public void deleteFailed(ManagedLedgerException ex, Object ctx) { + // The method internalMarkDelete already handled the failure operation. We only need to + // make sure the memory state is updated. + // If the broker crashed, the non-recoverable ledger will be detected again. + } + }, null); } // ////////////////////////////////////////////////// @@ -3613,4 +3617,29 @@ public boolean isCacheReadEntry() { public ManagedLedgerConfig getConfig() { return config; } + + /*** + * Create a non-durable cursor and copy the ack stats. + */ + public ManagedCursor duplicateNonDurableCursor(String nonDurableCursorName) throws ManagedLedgerException { + NonDurableCursorImpl newNonDurableCursor = + (NonDurableCursorImpl) ledger.newNonDurableCursor(getMarkDeletedPosition(), nonDurableCursorName); + if (individualDeletedMessages != null) { + this.individualDeletedMessages.forEach(range -> { + newNonDurableCursor.individualDeletedMessages.addOpenClosed( + range.lowerEndpoint().getLedgerId(), + range.lowerEndpoint().getEntryId(), + range.upperEndpoint().getLedgerId(), + range.upperEndpoint().getEntryId()); + return true; + }); + } + if (batchDeletedIndexes != null) { + for (Map.Entry entry : this.batchDeletedIndexes.entrySet()) { + BitSetRecyclable copiedBitSet = BitSetRecyclable.valueOf(entry.getValue()); + newNonDurableCursor.batchDeletedIndexes.put(entry.getKey(), copiedBitSet); + } + } + return newNonDurableCursor; + } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryMBeanImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryMBeanImpl.java index cf3d7142d617e..5a6bc8017b7e0 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryMBeanImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryMBeanImpl.java @@ -18,6 +18,7 @@ */ package org.apache.bookkeeper.mledger.impl; +import static com.google.common.base.Preconditions.checkArgument; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; import org.apache.bookkeeper.mledger.ManagedLedgerFactoryMXBean; @@ -41,6 +42,7 @@ public ManagedLedgerFactoryMBeanImpl(ManagedLedgerFactoryImpl factory) throws Ex } public void refreshStats(long period, TimeUnit unit) { + checkArgument(period >= 0); double seconds = unit.toMillis(period) / 1000.0; if (seconds <= 0.0) { diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 8ce2a6924ebed..75ac4dd4c0a44 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -3242,7 +3242,7 @@ void offloadLoop(CompletableFuture promise, Queue ledg .thenCompose(readHandle -> config.getLedgerOffloader().offload(readHandle, uuid, extraMetadata)) .thenCompose((ignore) -> { return Retries.run(Backoff.exponentialJittered(TimeUnit.SECONDS.toMillis(1), - TimeUnit.SECONDS.toHours(1)).limit(10), + TimeUnit.HOURS.toMillis(1)).limit(10), FAIL_ON_CONFLICT, () -> completeLedgerInfoForOffloaded(ledgerId, uuid), scheduledExecutor, name) @@ -4459,7 +4459,8 @@ private void cancelScheduledTasks() { @Override public boolean checkInactiveLedgerAndRollOver() { long currentTimeMs = System.currentTimeMillis(); - if (inactiveLedgerRollOverTimeMs > 0 && currentTimeMs > (lastAddEntryTimeMs + inactiveLedgerRollOverTimeMs)) { + if (currentLedgerEntries > 0 && inactiveLedgerRollOverTimeMs > 0 && currentTimeMs > (lastAddEntryTimeMs + + inactiveLedgerRollOverTimeMs)) { log.info("[{}] Closing inactive ledger, last-add entry {}", name, lastAddEntryTimeMs); if (STATE_UPDATER.compareAndSet(this, State.LedgerOpened, State.ClosingLedger)) { LedgerHandle currentLedger = this.currentLedger; @@ -4537,4 +4538,4 @@ public Position getTheSlowestNonDurationReadPosition() { } return theSlowestNonDurableReadPosition; } -} +} \ No newline at end of file diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java index e057dee99538e..7884add95526c 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java @@ -18,6 +18,7 @@ */ package org.apache.bookkeeper.mledger.impl; +import static com.google.common.base.Preconditions.checkArgument; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; import org.apache.bookkeeper.mledger.ManagedCursor; @@ -63,6 +64,7 @@ public ManagedLedgerMBeanImpl(ManagedLedgerImpl managedLedger) { } public void refreshStats(long period, TimeUnit unit) { + checkArgument(period >= 0); double seconds = unit.toMillis(period) / 1000.0; if (seconds <= 0.0) { // skip refreshing stats diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorImpl.java index 9102339b2904e..1661613f07d7d 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorImpl.java @@ -23,6 +23,7 @@ import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.ReadOnlyCursor; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.PositionBound; import org.apache.bookkeeper.mledger.proto.MLDataFormats; @@ -70,4 +71,9 @@ public MLDataFormats.ManagedLedgerInfo.LedgerInfo getCurrentLedgerInfo() { public long getNumberOfEntries(Range range) { return this.ledger.getNumberOfEntries(range); } + + @Override + public boolean isMessageDeleted(Position position) { + return false; + } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloadUtils.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloadUtils.java index 550626f76c000..9c9feb2aa7f7c 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloadUtils.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloadUtils.java @@ -198,7 +198,7 @@ public static CompletableFuture cleanupOffloaded(long ledgerId, UUID uuid, metadataMap.put("ManagedLedgerName", name); return Retries.run(Backoff.exponentialJittered(TimeUnit.SECONDS.toMillis(1), - TimeUnit.SECONDS.toHours(1)).limit(10), + TimeUnit.HOURS.toMillis(1)).limit(10), Retries.NonFatalPredicate, () -> mlConfig.getLedgerOffloader().deleteOffloaded(ledgerId, uuid, metadataMap), executor, name).whenComplete((ignored, exception) -> { diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index 627ae73d928bd..644f53c3a522d 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -43,6 +43,7 @@ import java.util.Arrays; import java.util.BitSet; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -65,6 +66,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; import lombok.Cleanup; @@ -72,6 +74,7 @@ import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.BookKeeper.DigestType; import org.apache.bookkeeper.client.LedgerEntry; +import org.apache.bookkeeper.client.api.ReadHandle; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCallback; @@ -766,7 +769,7 @@ void testResetCursor() throws Exception { @Test(timeOut = 20000) void testResetCursor1() throws Exception { ManagedLedger ledger = factory.open("my_test_move_cursor_ledger", - new ManagedLedgerConfig().setMaxEntriesPerLedger(2)); + new ManagedLedgerConfig().setMaxEntriesPerLedger(2)); ManagedCursor cursor = ledger.openCursor("trc1"); PositionImpl actualEarliest = (PositionImpl) ledger.addEntry("dummy-entry-1".getBytes(Encoding)); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); @@ -2286,7 +2289,7 @@ void testFindNewestMatchingEdgeCase1() throws Exception { ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1"); assertNull(c1.findNewestMatching( - entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding)))); + entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding)))); } @Test(timeOut = 20000) @@ -2595,7 +2598,7 @@ public void findEntryComplete(Position position, Object ctx) { @Override public void findEntryFailed(ManagedLedgerException exception, Optional failedReadPosition, - Object ctx) { + Object ctx) { result.exception = exception; counter.countDown(); } @@ -2621,7 +2624,7 @@ public void findEntryFailed(ManagedLedgerException exception, Optional } void internalTestFindNewestMatchingAllEntries(final String name, final int entriesPerLedger, - final int expectedEntryId) throws Exception { + final int expectedEntryId) throws Exception { final String ledgerAndCursorName = name; ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setRetentionSizeInMB(10); @@ -2715,7 +2718,7 @@ void testReplayEntries() throws Exception { assertTrue((Arrays.equals(entries.get(0).getData(), "entry1".getBytes(Encoding)) && Arrays.equals(entries.get(1).getData(), "entry3".getBytes(Encoding))) || (Arrays.equals(entries.get(0).getData(), "entry3".getBytes(Encoding)) - && Arrays.equals(entries.get(1).getData(), "entry1".getBytes(Encoding)))); + && Arrays.equals(entries.get(1).getData(), "entry1".getBytes(Encoding)))); entries.forEach(Entry::release); // 3. Fail on reading non-existing position @@ -3142,7 +3145,7 @@ public void operationFailed(ManagedLedgerException exception) { try { bkc.openLedgerNoRecovery(ledgerId, DigestType.fromApiDigestType(mlConfig.getDigestType()), - mlConfig.getPassword()); + mlConfig.getPassword()); fail("ledger should have deleted due to update-cursor failure"); } catch (BKException e) { // ok @@ -3761,17 +3764,17 @@ private void deleteBatchIndex(ManagedCursor cursor, Position position, int batch pos.ackSet = bitSet.toLongArray(); cursor.asyncDelete(pos, - new DeleteCallback() { - @Override - public void deleteComplete(Object ctx) { - latch.countDown(); - } + new DeleteCallback() { + @Override + public void deleteComplete(Object ctx) { + latch.countDown(); + } - @Override - public void deleteFailed(ManagedLedgerException exception, Object ctx) { - latch.countDown(); - } - }, null); + @Override + public void deleteFailed(ManagedLedgerException exception, Object ctx) { + latch.countDown(); + } + }, null); latch.await(); pos.ackSet = null; } @@ -4484,5 +4487,202 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { ledger.close(); } + + @Test + public void testReadEntriesWithSkipDeletedEntries() throws Exception { + @Cleanup + ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testReadEntriesWithSkipDeletedEntries"); + ledger = Mockito.spy(ledger); + List actualReadEntryIds = new ArrayList<>(); + Mockito.doAnswer(inv -> { + long start = inv.getArgument(1); + long end = inv.getArgument(2); + for (long i = start; i <= end; i++) { + actualReadEntryIds.add(i); + } + return inv.callRealMethod(); + }) + .when(ledger) + .asyncReadEntry(Mockito.any(ReadHandle.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + @Cleanup + ManagedCursor cursor = ledger.openCursor("c"); + + int entries = 20; + Position maxReadPosition = null; + Map map = new HashMap<>(); + for (int i = 0; i < entries; i++) { + maxReadPosition = ledger.addEntry(new byte[1024]); + map.put(i, maxReadPosition); + } + + + Set deletedPositions = new HashSet<>(); + deletedPositions.add(map.get(1)); + deletedPositions.add(map.get(4)); + deletedPositions.add(map.get(5)); + deletedPositions.add(map.get(8)); + deletedPositions.add(map.get(9)); + deletedPositions.add(map.get(10)); + deletedPositions.add(map.get(15)); + deletedPositions.add(map.get(17)); + deletedPositions.add(map.get(19)); + cursor.delete(deletedPositions); + + CompletableFuture f0 = new CompletableFuture<>(); + List readEntries = new ArrayList<>(); + cursor.asyncReadEntries(5, -1L, new ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + readEntries.addAll(entries); + f0.complete(null); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + f0.completeExceptionally(exception); + } + }, null, PositionImpl.get(maxReadPosition.getLedgerId(), maxReadPosition.getEntryId()).getNext()); + + f0.get(); + + CompletableFuture f1 = new CompletableFuture<>(); + cursor.asyncReadEntries(5, -1L, new ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + readEntries.addAll(entries); + f1.complete(null); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + f1.completeExceptionally(exception); + } + }, null, PositionImpl.get(maxReadPosition.getLedgerId(), maxReadPosition.getEntryId()).getNext()); + + + f1.get(); + CompletableFuture f2 = new CompletableFuture<>(); + cursor.asyncReadEntries(100, -1L, new ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + readEntries.addAll(entries); + f2.complete(null); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + f2.completeExceptionally(exception); + } + }, null, PositionImpl.get(maxReadPosition.getLedgerId(), maxReadPosition.getEntryId()).getNext()); + + f2.get(); + + Position cursorReadPosition = cursor.getReadPosition(); + Position expectReadPosition = maxReadPosition.getNext(); + assertTrue(cursorReadPosition.getLedgerId() == expectReadPosition.getLedgerId() + && cursorReadPosition.getEntryId() == expectReadPosition.getEntryId()); + + assertEquals(readEntries.size(), actualReadEntryIds.size()); + assertEquals(entries - deletedPositions.size(), actualReadEntryIds.size()); + for (Entry entry : readEntries) { + long entryId = entry.getEntryId(); + assertTrue(actualReadEntryIds.contains(entryId)); + } + } + + + @Test + public void testReadEntriesWithSkipDeletedEntriesAndWithSkipConditions() throws Exception { + @Cleanup + ManagedLedgerImpl ledger = (ManagedLedgerImpl) + factory.open("testReadEntriesWithSkipDeletedEntriesAndWithSkipConditions"); + ledger = Mockito.spy(ledger); + + List actualReadEntryIds = new ArrayList<>(); + Mockito.doAnswer(inv -> { + long start = inv.getArgument(1); + long end = inv.getArgument(2); + for (long i = start; i <= end; i++) { + actualReadEntryIds.add(i); + } + return inv.callRealMethod(); + }) + .when(ledger) + .asyncReadEntry(Mockito.any(ReadHandle.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + @Cleanup + ManagedCursor cursor = ledger.openCursor("c"); + + int entries = 20; + Position maxReadPosition0 = null; + Map map = new HashMap<>(); + for (int i = 0; i < entries; i++) { + maxReadPosition0 = ledger.addEntry(new byte[1024]); + map.put(i, maxReadPosition0); + } + + PositionImpl maxReadPosition = + PositionImpl.get(maxReadPosition0.getLedgerId(), maxReadPosition0.getEntryId()).getNext(); + + Set deletedPositions = new HashSet<>(); + deletedPositions.add(map.get(1)); + deletedPositions.add(map.get(3)); + deletedPositions.add(map.get(5)); + cursor.delete(deletedPositions); + + Set skippedPositions = new HashSet<>(); + skippedPositions.add(map.get(6).getEntryId()); + skippedPositions.add(map.get(7).getEntryId()); + skippedPositions.add(map.get(8).getEntryId()); + skippedPositions.add(map.get(11).getEntryId()); + skippedPositions.add(map.get(15).getEntryId()); + skippedPositions.add(map.get(16).getEntryId()); + + Predicate skipCondition = position -> skippedPositions.contains(position.getEntryId()); + List readEntries = new ArrayList<>(); + + CompletableFuture f0 = new CompletableFuture<>(); + cursor.asyncReadEntriesWithSkip(10, -1L, new ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + readEntries.addAll(entries); + f0.complete(null); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + f0.completeExceptionally(exception); + } + }, null, maxReadPosition, skipCondition); + + f0.get(); + CompletableFuture f1 = new CompletableFuture<>(); + cursor.asyncReadEntriesWithSkip(100, -1L, new ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + readEntries.addAll(entries); + f1.complete(null); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + f1.completeExceptionally(exception); + } + }, null, maxReadPosition, skipCondition); + f1.get(); + + + assertEquals(actualReadEntryIds.size(), readEntries.size()); + assertEquals(entries - deletedPositions.size() - skippedPositions.size(), actualReadEntryIds.size()); + for (Entry entry : readEntries) { + long entryId = entry.getEntryId(); + assertTrue(actualReadEntryIds.contains(entryId)); + } + + Position cursorReadPosition = cursor.getReadPosition(); + Position expectReadPosition = maxReadPosition; + assertTrue(cursorReadPosition.getLedgerId() == expectReadPosition.getLedgerId() + && cursorReadPosition.getEntryId() == expectReadPosition.getEntryId()); + } + private static final Logger log = LoggerFactory.getLogger(ManagedCursorTest.class); } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index 8430afb4e4f82..4c92911c687f5 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -3898,6 +3898,30 @@ public void testInactiveLedgerRollOver() throws Exception { ledger.close(); } + @Test + public void testDontRollOverEmptyInactiveLedgers() throws Exception { + int inactiveLedgerRollOverTimeMs = 5; + ManagedLedgerFactoryConfig factoryConf = new ManagedLedgerFactoryConfig(); + @Cleanup("shutdown") + ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc); + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setInactiveLedgerRollOverTime(inactiveLedgerRollOverTimeMs, TimeUnit.MILLISECONDS); + ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("rollover_inactive", config); + ManagedCursor cursor = ledger.openCursor("c1"); + + long ledgerId = ledger.currentLedger.getId(); + + Thread.sleep(inactiveLedgerRollOverTimeMs * 5); + ledger.checkInactiveLedgerAndRollOver(); + + Thread.sleep(inactiveLedgerRollOverTimeMs * 5); + ledger.checkInactiveLedgerAndRollOver(); + + assertEquals(ledger.currentLedger.getId(), ledgerId); + + ledger.close(); + } + @Test public void testOffloadTaskCancelled() throws Exception { @Cleanup("shutdown") @@ -4094,6 +4118,7 @@ public void testNonDurableCursorCreateForInactiveLedger() throws Exception { ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setInactiveLedgerRollOverTime(10, TimeUnit.MILLISECONDS); ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open(mlName, config); + ml.addEntry("entry".getBytes(UTF_8)); MutableBoolean isRolledOver = new MutableBoolean(false); retryStrategically((test) -> { diff --git a/microbench/pom.xml b/microbench/pom.xml index a62876e8802f7..a175c761bf184 100644 --- a/microbench/pom.xml +++ b/microbench/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT ../pom.xml - microbench jar Pulsar Microbenchmarks - 1.37 - microbench @@ -79,7 +75,6 @@ - org.openjdk.jmh @@ -98,7 +93,6 @@ ${project.version} - @@ -130,4 +124,4 @@ - \ No newline at end of file + diff --git a/microbench/src/main/resources/log4j2.xml b/microbench/src/main/resources/log4j2.xml index 7ec5ed8169a66..c48941ef785e5 100644 --- a/microbench/src/main/resources/log4j2.xml +++ b/microbench/src/main/resources/log4j2.xml @@ -1,4 +1,4 @@ - + - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index 6844f20e1aa9e..bc04cd2d09841 100644 --- a/pom.xml +++ b/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 pom @@ -28,30 +27,24 @@ apache 29 - - org.apache.pulsar + io.streamnative pulsar - 3.2.0-SNAPSHOT - Pulsar Pulsar is a distributed pub-sub messaging platform with a very flexible messaging model and an intuitive client API. https://github.com/apache/pulsar - Apache Software Foundation https://www.apache.org/ 2017 - Apache Pulsar developers https://pulsar.apache.org/ - Apache License, Version 2.0 @@ -59,47 +52,38 @@ flexible messaging model and an intuitive client API. repo - - https://github.com/apache/pulsar - scm:git:https://github.com/apache/pulsar.git - scm:git:ssh://git@github.com:apache/pulsar.git + https://github.com/streamnative/pulsar + scm:git:https://github.com/streamnative/pulsar.git + scm:git:ssh://git@github.com:streamnative/pulsar.git - GitHub Actions https://github.com/apache/pulsar/actions - Github https://github.com/apache/pulsar/issues - 17 17 ${maven.compiler.target} 8 - - 3.2.0 - + 3.4.0 **/Test*.java,**/*Test.java,**/*Tests.java,**/*TestCase.java quarantine - UTF-8 UTF-8 - 2023-09-08T03:29:19Z + 2024-03-04T21:08:59Z true - - - + --add-opens java.base/jdk.internal.loader=ALL-UNNAMED @@ -116,7 +100,7 @@ flexible messaging model and an intuitive client API. false 1 true - + false ${project.build.directory} @@ -129,20 +113,20 @@ flexible messaging model and an intuitive client API. false package package - - 1.21 - - 4.16.3 + 1.26.0 + 4.16.4 3.9.1 1.5.0 1.10.0 - 1.1.10.5 - 4.1.12.1 + 1.1.10.5 + + 4.1.12.1 + 5.1.0 4.1.104.Final 0.0.24.Final - 9.4.53.v20231009 + 9.4.54.v20240208 2.5.2 2.34 1.10.50 @@ -162,6 +146,8 @@ flexible messaging model and an intuitive client API. 0.43.3 true 0.5.0 + 1.14.12 + 1.17 3.19.6 ${protobuf3.version} 1.55.3 @@ -248,11 +234,9 @@ flexible messaging model and an intuitive client API. 3.4.3 1.5.2-3 2.0.6 - 1.18.3 2.2 - 3.3.0 1.1.1 @@ -260,12 +244,12 @@ flexible messaging model and an intuitive client API. 5.6.0 3.25.0-GA 1.5.0 - 3.3 + 3.3 + 4.2.0 1.5.4 5.4.0 2.33.2 - 0.6.1 3.0.0 @@ -300,20 +284,16 @@ flexible messaging model and an intuitive client API. 1.6.1 6.4.0 3.33.0 - rename-netty-native-libs.sh - - org.jline jline ${jline3.version} - org.asynchttpclient async-http-client @@ -329,39 +309,34 @@ flexible messaging model and an intuitive client API. - org.testng testng ${testng.version} - - org.yaml - * - + + org.yaml + * + - org.hamcrest hamcrest ${hamcrest.version} test - org.awaitility awaitility ${awaitility.version} test - org.mockito mockito-core ${mockito.version} - org.apache.zookeeper zookeeper @@ -437,7 +412,6 @@ flexible messaging model and an intuitive client API. - org.apache.bookkeeper bookkeeper-server @@ -466,13 +440,11 @@ flexible messaging model and an intuitive client API. - org.apache.bookkeeper cpu-affinity ${bookkeeper.version} - io.vertx vertx-core @@ -483,38 +455,33 @@ flexible messaging model and an intuitive client API. vertx-web ${vertx.version} - - org.apache.curator - curator-recipes - ${curator.version} - - - org.apache.zookeeper - * - - + org.apache.curator + curator-recipes + ${curator.version} + + + org.apache.zookeeper + * + + - org.apache.bookkeeper bookkeeper-common-allocator ${bookkeeper.version} - org.apache.bookkeeper bookkeeper-tools-framework ${bookkeeper.version} - org.reflections reflections ${reflections.version} - org.apache.bookkeeper @@ -547,7 +514,6 @@ flexible messaging model and an intuitive client API. - org.apache.bookkeeper @@ -580,19 +546,16 @@ flexible messaging model and an intuitive client API. - org.apache.bookkeeper bookkeeper-common ${bookkeeper.version} - org.apache.bookkeeper.stats bookkeeper-stats-api ${bookkeeper.version} - org.apache.bookkeeper.stats datasketches-metrics-provider @@ -604,37 +567,31 @@ flexible messaging model and an intuitive client API. - org.apache.bookkeeper.stats prometheus-metrics-provider ${bookkeeper.version} - org.rocksdb rocksdbjni ${rocksdb.version} - org.eclipse.jetty jetty-server ${jetty.version} - org.eclipse.jetty jetty-alpn-conscrypt-server ${jetty.version} - org.conscrypt conscrypt-openjdk-uber ${conscrypt.version} - org.eclipse.jetty jetty-bom @@ -642,7 +599,6 @@ flexible messaging model and an intuitive client API. pom import - io.netty netty-bom @@ -650,7 +606,6 @@ flexible messaging model and an intuitive client API. pom import - io.netty.incubator netty-incubator-transport-classes-io_uring @@ -673,79 +628,66 @@ flexible messaging model and an intuitive client API. ${netty-iouring.version} linux-aarch_64 - com.beust jcommander ${jcommander.version} - com.google.guava guava ${guava.version} - com.google.inject guice ${guice.version} - com.google.inject.extensions guice-assistedinject ${guice.version} - org.apache.commons commons-lang3 ${commons-lang3.version} - org.apache.commons commons-compress ${commons-compress.version} - commons-configuration commons-configuration ${commons-configuration.version} - commons-io commons-io ${commons-io.version} - org.apache.commons commons-text ${commons-text.version} - org.slf4j slf4j-api ${slf4j.version} - org.slf4j slf4j-simple ${slf4j.version} - org.slf4j jcl-over-slf4j ${slf4j.version} - org.apache.logging.log4j log4j-bom @@ -753,49 +695,41 @@ flexible messaging model and an intuitive client API. pom import - commons-codec commons-codec ${commons-codec.version} - org.glassfish.jersey.core jersey-server ${jersey.version} - org.glassfish.jersey.core jersey-client ${jersey.version} - org.glassfish.jersey.inject jersey-hk2 ${jersey.version} - org.glassfish.jersey.containers jersey-container-servlet-core ${jersey.version} - org.glassfish.jersey.containers jersey-container-servlet ${jersey.version} - javax.ws.rs javax.ws.rs-api ${javax.ws.rs-api.version} - org.glassfish.jersey.media jersey-media-json-jackson @@ -807,19 +741,16 @@ flexible messaging model and an intuitive client API. - org.glassfish.jersey.media jersey-media-multipart ${jersey.version} - net.java.dev.jna jna ${jna.version} - com.github.docker-java docker-java-core @@ -841,7 +772,6 @@ flexible messaging model and an intuitive client API. docker-java-transport-zerodep ${docker-java.version} - com.fasterxml.jackson jackson-bom @@ -849,56 +779,46 @@ flexible messaging model and an intuitive client API. pom import - org.codehaus.jettison jettison ${jettison.version} - com.fasterxml.woodstox woodstox-core ${woodstox.version} - - org.hdrhistogram HdrHistogram ${hdrHistogram.version} - io.swagger swagger-core ${swagger.version} - io.swagger swagger-annotations ${swagger.version} - javax.servlet javax.servlet-api ${javax.servlet-api} - com.github.ben-manes.caffeine caffeine ${caffeine.version} - org.bouncycastle bcpkix-jdk18on ${bouncycastle.version} - com.cronutils cron-utils @@ -910,19 +830,16 @@ flexible messaging model and an intuitive client API. - com.yahoo.athenz athenz-zts-java-client-core ${athenz.version} - com.yahoo.athenz athenz-zpe-java-client ${athenz.version} - com.yahoo.athenz athenz-cert-refresher @@ -934,7 +851,6 @@ flexible messaging model and an intuitive client API. - com.yahoo.athenz athenz-auth-core @@ -946,67 +862,56 @@ flexible messaging model and an intuitive client API. - com.github.zafarkhaja java-semver ${java-semver.version} - io.prometheus simpleclient ${prometheus.version} - io.prometheus simpleclient_hotspot ${prometheus.version} - io.prometheus simpleclient_log4j2 ${prometheus.version} - io.prometheus simpleclient_servlet ${prometheus.version} - io.prometheus simpleclient_jetty ${prometheus.version} - io.prometheus simpleclient_caffeine ${prometheus.version} - com.carrotsearch hppc ${hppc.version} - io.etcd jetcd-core ${jetcd.version} - io.etcd jetcd-test ${jetcd.version} - org.apache.spark spark-streaming_2.10 @@ -1034,7 +939,6 @@ flexible messaging model and an intuitive client API. - io.jsonwebtoken jjwt-api @@ -1050,13 +954,21 @@ flexible messaging model and an intuitive client API. jjwt-jackson ${jsonwebtoken.version} - net.jodah typetools ${typetools.version} - + + net.bytebuddy + byte-buddy + ${byte-buddy.version} + + + org.zeroturnaround + zt-zip + ${zt-zip.version} + io.grpc grpc-bom @@ -1064,7 +976,6 @@ flexible messaging model and an intuitive client API. pom import - io.grpc grpc-all @@ -1088,7 +999,6 @@ flexible messaging model and an intuitive client API. - io.grpc grpc-xds @@ -1100,25 +1010,21 @@ flexible messaging model and an intuitive client API. - com.google.http-client google-http-client ${google-http-client.version} - com.google.http-client google-http-client-jackson2 ${google-http-client.version} - com.google.http-client google-http-client-gson ${google-http-client.version} - io.perfmark perfmark-api @@ -1131,7 +1037,6 @@ flexible messaging model and an intuitive client API. - com.google.protobuf protobuf-bom @@ -1139,19 +1044,16 @@ flexible messaging model and an intuitive client API. pom import - com.google.code.gson gson ${gson.version} - com.yahoo.datasketches sketches-core ${sketches.version} - com.amazonaws aws-java-sdk-bom @@ -1159,7 +1061,6 @@ flexible messaging model and an intuitive client API. pom import - org.apache.distributedlog distributedlog-core @@ -1172,13 +1073,11 @@ flexible messaging model and an intuitive client API. - org.apache.commons commons-collections4 ${commons.collections4.version} - com.lmax @@ -1202,103 +1101,86 @@ flexible messaging model and an intuitive client API. assertj-core ${assertj-core.version} - org.projectlombok lombok ${lombok.version} - javax.annotation javax.annotation-api ${javax.annotation-api.version} - javax.xml.bind jaxb-api ${jaxb-api} - jakarta.xml.bind jakarta.xml.bind-api ${jakarta.xml.bind.version} - com.sun.activation javax.activation ${javax.activation.version} - com.sun.activation jakarta.activation ${jakarta.activation.version} - jakarta.activation jakarta.activation-api ${jakarta.activation.version} - jakarta.validation jakarta.validation-api ${jakarta.validation.version} - io.opencensus opencensus-api ${opencensus.version} - io.opencensus opencensus-contrib-http-util ${opencensus.version} - io.opencensus opencensus-contrib-grpc-metrics ${opencensus.version} - org.opensearch.client opensearch-rest-high-level-client ${opensearch.version} - co.elastic.clients elasticsearch-java ${elasticsearch-java.version} - joda-time joda-time ${joda.version} - org.javassist javassist ${javassist.version} - net.jcip jcip-annotations ${jcip.version} - io.airlift aircompressor @@ -1310,25 +1192,21 @@ flexible messaging model and an intuitive client API. - org.objenesis objenesis ${objenesis.version} - org.apache.httpcomponents httpclient ${apache-http-client.version} - org.apache.httpcomponents httpcore ${apache-httpcomponents.version} - com.github.spotbugs spotbugs-annotations @@ -1336,31 +1214,26 @@ flexible messaging model and an intuitive client API. provided true - com.google.errorprone error_prone_annotations ${errorprone.version} - com.google.j2objc j2objc-annotations ${j2objc-annotations.version} - org.yaml snakeyaml ${snakeyaml.version} - org.apache.ant ant ${ant.version} - com.squareup.okhttp3 okhttp @@ -1381,7 +1254,6 @@ flexible messaging model and an intuitive client API. okio ${okio.version} - org.jetbrains.kotlin kotlin-stdlib @@ -1392,25 +1264,21 @@ flexible messaging model and an intuitive client API. kotlin-stdlib-common ${kotlin-stdlib.version} - org.jetbrains.kotlin kotlin-stdlib-jdk8 ${kotlin-stdlib.version} - com.github.luben zstd-jni ${zstd-jni.version} - - com.typesafe.netty - netty-reactive-streams - ${netty-reactive-streams.version} + com.typesafe.netty + netty-reactive-streams + ${netty-reactive-streams.version} - org.roaringbitmap RoaringBitmap @@ -1428,41 +1296,35 @@ flexible messaging model and an intuitive client API. - - org.apache.pulsar + io.streamnative buildtools ${project.version} test - org.testng testng test - org.mockito mockito-core test - com.github.stefanbirkner system-lambda ${system-lambda.version} test - org.assertj assertj-core test - org.projectlombok lombok @@ -1473,7 +1335,6 @@ flexible messaging model and an intuitive client API. javax.annotation-api provided - org.apache.bookkeeper @@ -1483,8 +1344,8 @@ flexible messaging model and an intuitive client API. tests - org.bouncycastle - * + org.bouncycastle + * org.slf4j @@ -1518,8 +1379,12 @@ flexible messaging model and an intuitive client API. + + org.apache.logging.log4j + log4j-layout-template-json + 2.20.0 + - ${project.artifactId} @@ -1594,7 +1459,6 @@ flexible messaging model and an intuitive client API. - org.commonjava.maven.plugins directory-maven-plugin @@ -1609,14 +1473,13 @@ flexible messaging model and an intuitive client API. pulsar.basedir - org.apache.pulsar + io.streamnative pulsar - pl.project13.maven git-commit-id-plugin @@ -1641,7 +1504,6 @@ flexible messaging model and an intuitive client API. - com.mycila license-maven-plugin @@ -1742,33 +1604,26 @@ flexible messaging model and an intuitive client API. src/assemble/README.bin.txt src/assemble/LICENSE.bin.txt src/assemble/NOTICE.bin.txt - src/main/java/org/apache/bookkeeper/mledger/proto/MLDataFormats.java src/main/java/org/apache/pulsar/broker/service/schema/proto/SchemaRegistryFormat.java bin/proto/MLDataFormats_pb2.py - **/avro/generated/*.java - **/*.avsc - src/main/java/org/apache/pulsar/io/kinesis/fbs/CompressionType.java src/main/java/org/apache/pulsar/io/kinesis/fbs/EncryptionCtx.java src/main/java/org/apache/pulsar/io/kinesis/fbs/EncryptionKey.java src/main/java/org/apache/pulsar/io/kinesis/fbs/KeyValue.java src/main/java/org/apache/pulsar/io/kinesis/fbs/Message.java - src/main/java/org/apache/bookkeeper/mledger/util/AbstractCASReferenceCounted.java - dependency-reduced-pom.xml - pulsar-client-go/go.mod pulsar-client-go/go.sum @@ -1776,14 +1631,11 @@ flexible messaging model and an intuitive client API. pulsar-function-go/go.sum pulsar-function-go/examples/go.mod pulsar-function-go/examples/go.sum - **/META-INF/services/com.scurrilous.circe.HashProvider - **/django/stats/migrations/*.py **/conf/uwsgi_params - **/*.crt **/*.key @@ -1798,21 +1650,16 @@ flexible messaging model and an intuitive client API. certificate-authority/index.txt certificate-authority/serial certificate-authority/README.md - **/zk-3.5-test-data/* - **/requirements.txt - conf/schema_example.json **/templates/*.tpl - **/.helmignore **/_helpers.tpl - **/*.md .github/** @@ -1841,6 +1688,7 @@ flexible messaging model and an intuitive client API. **/*.so.* **/*.dylib src/test/resources/*.txt + **/dependency-reduced-pom.xml @@ -1849,27 +1697,78 @@ flexible messaging model and an intuitive client API. maven-enforcer-plugin ${maven-enforcer-plugin.version} - - enforce-maven - - enforce - - - - - 17 - Java 17+ is required to build Pulsar. - - - 3.6.1 - - - - + + enforce-maven + + enforce + + + + + 17 + Java 17+ is required to build Pulsar. + + + 3.6.1 + + + + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.13 + true + + ossrh + https://s01.oss.sonatype.org/ + 60285aee9d4161 + true + + + + org.apache.maven.plugins + maven-source-plugin + 3.2.0 + + + attach-sources + + jar-no-fork + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + attach-javadocs + + jar + + + none + false + + + + + + org.apache.maven.plugins + maven-release-plugin + 3.0.0-M1 + + true + false + release + deploy + + - @@ -1890,10 +1789,10 @@ flexible messaging model and an intuitive client API. maven-surefire-plugin - ${include} + ${include} - **/*$*,${exclude} + **/*$*,${exclude} ${groups} ${excludedGroups} @@ -1977,13 +1876,13 @@ flexible messaging model and an intuitive client API. com.github.spotbugs spotbugs-maven-plugin ${spotbugs-maven-plugin.version} - - - com.github.spotbugs - spotbugs - ${spotbugs.version} - - + + + com.github.spotbugs + spotbugs + ${spotbugs.version} + + org.codehaus.mojo @@ -1995,6 +1894,16 @@ flexible messaging model and an intuitive client API. docker-maven-plugin ${docker-maven.version} + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.13 + + + org.apache.maven.plugins + maven-source-plugin + 3.2.0 + @@ -2010,7 +1919,6 @@ flexible messaging model and an intuitive client API. - @@ -2119,7 +2027,6 @@ flexible messaging model and an intuitive client API. tests - contrib-check - - - - org.apache.rat - apache-rat-plugin - - - - check - - verify - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - check-style - verify - - ${pulsar.basedir}/buildtools/src/main/resources/pulsar/checkstyle.xml - ${pulsar.basedir}/buildtools/src/main/resources/pulsar/suppressions.xml - UTF-8 - **/proto/* - - - check - - - - - - + + + + org.apache.rat + apache-rat-plugin + + + + check + + verify + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + check-style + verify + + ${pulsar.basedir}/buildtools/src/main/resources/pulsar/checkstyle.xml + ${pulsar.basedir}/buildtools/src/main/resources/pulsar/suppressions.xml + UTF-8 + **/proto/* + + + check + + + + + + - windows @@ -2175,7 +2081,6 @@ flexible messaging model and an intuitive client API. rename-netty-native-libs.cmd - @@ -2195,6 +2100,7 @@ flexible messaging model and an intuitive client API. managed-ledger tiered-storage pulsar-common + pulsar-bom pulsar-broker-common pulsar-broker pulsar-cli-utils @@ -2220,39 +2126,28 @@ flexible messaging model and an intuitive client API. pulsar-broker-auth-sasl pulsar-client-auth-sasl pulsar-config-validation - structured-event-log - pulsar-transaction - pulsar-functions - pulsar-io - bouncy-castle - pulsar-client-messagecrypto-bc - pulsar-metadata jclouds-shaded - pulsar-package-management - distribution docker tests - microbench - core-modules @@ -2264,6 +2159,7 @@ flexible messaging model and an intuitive client API. testmocks managed-ledger pulsar-common + pulsar-bom pulsar-broker-common pulsar-broker pulsar-cli-utils @@ -2283,31 +2179,23 @@ flexible messaging model and an intuitive client API. pulsar-broker-auth-sasl pulsar-client-auth-sasl pulsar-config-validation - pulsar-transaction - pulsar-functions - pulsar-io - bouncy-castle - pulsar-client-messagecrypto-bc - distribution pulsar-metadata - pulsar-package-management - + + 4.0.0 + pom + + org.apache + apache + 29 + + + io.streamnative + pulsar-bom + 3.2.0 + Pulsar BOM + Pulsar (Bill of Materials) + https://github.com/apache/pulsar + + Apache Software Foundation + https://www.apache.org/ + + 2017 + + + Apache Pulsar developers + https://pulsar.apache.org/ + + + + + Apache License, Version 2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + https://github.com/apache/pulsar + scm:git:https://github.com/apache/pulsar.git + scm:git:ssh://git@github.com:apache/pulsar.git + + + GitHub Actions + https://github.com/apache/pulsar/actions + + + Github + https://github.com/apache/pulsar/issues + + + 17 + 17 + UTF-8 + UTF-8 + 2024-03-04T21:08:59Z + 4.1 + 3.1.2 + 3.5.3 + + + + + com.mycila + license-maven-plugin + ${license-maven-plugin.version} + + + +
../src/license-header.txt
+ + + + + + org.apache.rat + apache-rat-plugin + + + + dependency-reduced-pom.xml + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + ${maven-checkstyle-plugin.version} + + true + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + + + + + org.apache.maven.wagon + wagon-ssh-external + ${wagon-ssh-external.version} + + + + + + + + + + io.streamnative + bouncy-castle-bc + ${project.version} + + + io.streamnative + bouncy-castle-bcfips + ${project.version} + + + io.streamnative + bouncy-castle-parent + ${project.version} + + + io.streamnative + buildtools + ${project.version} + + + io.streamnative + distribution + ${project.version} + + + io.streamnative + docker-images + ${project.version} + + + io.streamnative + jclouds-shaded + ${project.version} + + + io.streamnative + managed-ledger + ${project.version} + + + io.streamnative + pulsar-all-docker-image + ${project.version} + + + io.streamnative + pulsar-broker-auth-athenz + ${project.version} + + + io.streamnative + pulsar-broker-auth-oidc + ${project.version} + + + io.streamnative + pulsar-broker-auth-sasl + ${project.version} + + + io.streamnative + pulsar-broker-common + ${project.version} + + + io.streamnative + pulsar-broker + ${project.version} + + + io.streamnative + pulsar-cli-utils + ${project.version} + + + io.streamnative + pulsar-client-1x-base + ${project.version} + + + io.streamnative + pulsar-client-1x + ${project.version} + + + io.streamnative + pulsar-client-2x-shaded + ${project.version} + + + io.streamnative + pulsar-client-admin-api + ${project.version} + + + io.streamnative + pulsar-client-admin-original + ${project.version} + + + io.streamnative + pulsar-client-admin + ${project.version} + + + io.streamnative + pulsar-client-all + ${project.version} + + + io.streamnative + pulsar-client-api + ${project.version} + + + io.streamnative + pulsar-client-auth-athenz + ${project.version} + + + io.streamnative + pulsar-client-auth-sasl + ${project.version} + + + io.streamnative + pulsar-client-messagecrypto-bc + ${project.version} + + + io.streamnative + pulsar-client-original + ${project.version} + + + io.streamnative + pulsar-client-tools-api + ${project.version} + + + io.streamnative + pulsar-client-tools + ${project.version} + + + io.streamnative + pulsar-client + ${project.version} + + + io.streamnative + pulsar-common + ${project.version} + + + io.streamnative + pulsar-config-validation + ${project.version} + + + io.streamnative + pulsar-docker-image + ${project.version} + + + io.streamnative + pulsar-docs-tools + ${project.version} + + + io.streamnative + pulsar-functions-api-examples-builtin + ${project.version} + + + io.streamnative + pulsar-functions-api-examples + ${project.version} + + + io.streamnative + pulsar-functions-api + ${project.version} + + + io.streamnative + pulsar-functions-instance + ${project.version} + + + io.streamnative + pulsar-functions-local-runner-original + ${project.version} + + + io.streamnative + pulsar-functions-local-runner + ${project.version} + + + io.streamnative + pulsar-functions-proto + ${project.version} + + + io.streamnative + pulsar-functions-runtime-all + ${project.version} + + + io.streamnative + pulsar-functions-runtime + ${project.version} + + + io.streamnative + pulsar-functions-secrets + ${project.version} + + + io.streamnative + pulsar-functions-utils + ${project.version} + + + io.streamnative + pulsar-functions-worker + ${project.version} + + + io.streamnative + pulsar-functions + ${project.version} + + + io.streamnative + pulsar-io-aerospike + ${project.version} + + + io.streamnative + pulsar-io-alluxio + ${project.version} + + + io.streamnative + pulsar-io-aws + ${project.version} + + + io.streamnative + pulsar-io-batch-data-generator + ${project.version} + + + io.streamnative + pulsar-io-batch-discovery-triggerers + ${project.version} + + + io.streamnative + pulsar-io-canal + ${project.version} + + + io.streamnative + pulsar-io-cassandra + ${project.version} + + + io.streamnative + pulsar-io-common + ${project.version} + + + io.streamnative + pulsar-io-core + ${project.version} + + + io.streamnative + pulsar-io-data-generator + ${project.version} + + + io.streamnative + pulsar-io-debezium-core + ${project.version} + + + io.streamnative + pulsar-io-debezium-mongodb + ${project.version} + + + io.streamnative + pulsar-io-debezium-mssql + ${project.version} + + + io.streamnative + pulsar-io-debezium-mysql + ${project.version} + + + io.streamnative + pulsar-io-debezium-oracle + ${project.version} + + + io.streamnative + pulsar-io-debezium-postgres + ${project.version} + + + io.streamnative + pulsar-io-debezium + ${project.version} + + + io.streamnative + pulsar-io-distribution + ${project.version} + + + io.streamnative + pulsar-io-docs + ${project.version} + + + io.streamnative + pulsar-io-dynamodb + ${project.version} + + + io.streamnative + pulsar-io-elastic-search + ${project.version} + + + io.streamnative + pulsar-io-file + ${project.version} + + + io.streamnative + pulsar-io-flume + ${project.version} + + + io.streamnative + pulsar-io-hbase + ${project.version} + + + io.streamnative + pulsar-io-hdfs2 + ${project.version} + + + io.streamnative + pulsar-io-hdfs3 + ${project.version} + + + io.streamnative + pulsar-io-http + ${project.version} + + + io.streamnative + pulsar-io-influxdb + ${project.version} + + + io.streamnative + pulsar-io-jdbc-clickhouse + ${project.version} + + + io.streamnative + pulsar-io-jdbc-core + ${project.version} + + + io.streamnative + pulsar-io-jdbc-mariadb + ${project.version} + + + io.streamnative + pulsar-io-jdbc-openmldb + ${project.version} + + + io.streamnative + pulsar-io-jdbc-postgres + ${project.version} + + + io.streamnative + pulsar-io-jdbc-sqlite + ${project.version} + + + io.streamnative + pulsar-io-jdbc + ${project.version} + + + io.streamnative + pulsar-io-kafka-connect-adaptor-nar + ${project.version} + + + io.streamnative + pulsar-io-kafka-connect-adaptor + ${project.version} + + + io.streamnative + pulsar-io-kafka + ${project.version} + + + io.streamnative + pulsar-io-kinesis + ${project.version} + + + io.streamnative + pulsar-io-mongo + ${project.version} + + + io.streamnative + pulsar-io-netty + ${project.version} + + + io.streamnative + pulsar-io-nsq + ${project.version} + + + io.streamnative + pulsar-io-rabbitmq + ${project.version} + + + io.streamnative + pulsar-io-redis + ${project.version} + + + io.streamnative + pulsar-io-solr + ${project.version} + + + io.streamnative + pulsar-io-twitter + ${project.version} + + + io.streamnative + pulsar-io + ${project.version} + + + io.streamnative + pulsar-metadata + ${project.version} + + + io.streamnative + pulsar-offloader-distribution + ${project.version} + + + io.streamnative + pulsar-package-bookkeeper-storage + ${project.version} + + + io.streamnative + pulsar-package-core + ${project.version} + + + io.streamnative + pulsar-package-filesystem-storage + ${project.version} + + + io.streamnative + pulsar-package-management + ${project.version} + + + io.streamnative + pulsar-proxy + ${project.version} + + + io.streamnative + pulsar-server-distribution + ${project.version} + + + io.streamnative + pulsar-shell-distribution + ${project.version} + + + io.streamnative + pulsar-testclient + ${project.version} + + + io.streamnative + pulsar-transaction-common + ${project.version} + + + io.streamnative + pulsar-transaction-coordinator + ${project.version} + + + io.streamnative + pulsar-transaction-parent + ${project.version} + + + io.streamnative + pulsar-websocket + ${project.version} + + + io.streamnative + pulsar + ${project.version} + + + io.streamnative + structured-event-log + ${project.version} + + + io.streamnative + testmocks + ${project.version} + + + io.streamnative + tiered-storage-file-system + ${project.version} + + + io.streamnative + tiered-storage-jcloud + ${project.version} + + + io.streamnative + tiered-storage-parent + ${project.version} + + + + + + ossrh + https://s01.oss.sonatype.org/content/repositories/snapshots + + + ossrh + https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ + + + diff --git a/pulsar-broker-auth-athenz/pom.xml b/pulsar-broker-auth-athenz/pom.xml index 6d1c14444ee9e..8a851f1257b1d 100644 --- a/pulsar-broker-auth-athenz/pom.xml +++ b/pulsar-broker-auth-athenz/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT - pulsar-broker-auth-athenz jar Athenz authentication plugin for broker - - ${project.groupId} pulsar-broker ${project.version} - ${project.groupId} testmocks ${project.version} test - com.yahoo.athenz athenz-zpe-java-client - org.bouncycastle bcpkix-jdk18on - - @@ -79,7 +69,6 @@ - com.github.spotbugs spotbugs-maven-plugin @@ -97,7 +86,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -111,7 +99,6 @@ - diff --git a/pulsar-broker-auth-athenz/src/test/resources/findbugsExclude.xml b/pulsar-broker-auth-athenz/src/test/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-broker-auth-athenz/src/test/resources/findbugsExclude.xml +++ b/pulsar-broker-auth-athenz/src/test/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-broker-auth-oidc/pom.xml b/pulsar-broker-auth-oidc/pom.xml index 1bc1abc579a38..74b59787c552c 100644 --- a/pulsar-broker-auth-oidc/pom.xml +++ b/pulsar-broker-auth-oidc/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT - pulsar-broker-auth-oidc jar Open ID Connect authentication plugin for broker - 0.11.5 - - ${project.groupId} pulsar-broker-common @@ -50,29 +44,24 @@ - com.auth0 java-jwt 4.3.0 - com.auth0 jwks-rsa 0.22.0 - com.github.ben-manes.caffeine caffeine - org.asynchttpclient async-http-client - io.kubernetes client-java @@ -97,7 +86,6 @@ - io.jsonwebtoken jjwt-api @@ -110,16 +98,13 @@ ${jsonwebtoken.version} test - com.github.tomakehurst wiremock-jre8 ${wiremock.version} test - - @@ -141,8 +126,6 @@ - - @@ -162,7 +145,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-broker-auth-sasl/pom.xml b/pulsar-broker-auth-sasl/pom.xml index dc8c3dfafa43c..47833dc38095c 100644 --- a/pulsar-broker-auth-sasl/pom.xml +++ b/pulsar-broker-auth-sasl/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT - pulsar-broker-auth-sasl jar SASL authentication plugin for broker - - ${project.groupId} pulsar-broker ${project.version} - org.apache.kerby kerby-config @@ -53,7 +47,6 @@ - org.apache.kerby kerb-simplekdc @@ -66,30 +59,25 @@ - ${project.groupId} pulsar-proxy ${project.version} test - ${project.groupId} testmocks ${project.version} test - ${project.groupId} pulsar-client-auth-sasl ${project.version} test - - @@ -111,8 +99,6 @@ - - @@ -132,7 +118,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-broker-common/pom.xml b/pulsar-broker-common/pom.xml index bf41c63f5f67d..c2c1ceb56f366 100644 --- a/pulsar-broker-common/pom.xml +++ b/pulsar-broker-common/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT - pulsar-broker-common Common classes used in multiple broker modules - ${project.groupId} pulsar-metadata ${project.version} - com.google.guava guava - io.prometheus simpleclient_jetty - javax.servlet javax.servlet-api - javax.ws.rs javax.ws.rs-api - io.jsonwebtoken jjwt-impl - io.jsonwebtoken jjwt-jackson - org.bouncycastle @@ -76,14 +65,12 @@ ${bouncycastle.bc-fips.version} test - org.awaitility awaitility test - @@ -103,7 +90,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -117,7 +103,6 @@ - maven-resources-plugin diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TopicResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TopicResources.java index 840ced0a1c1c4..0963f25c3d31f 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TopicResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TopicResources.java @@ -50,6 +50,9 @@ public TopicResources(MetadataStore store) { store.registerListener(this::handleNotification); } + /*** + * List persistent topics names under a namespace, the topic name contains the partition suffix. + */ public CompletableFuture> listPersistentTopicsAsync(NamespaceName ns) { String path = MANAGED_LEDGER_PATH + "/" + ns + "/persistent"; diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServlets.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServlets.java index 0046d28afa444..f6fc42ff0fccf 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServlets.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServlets.java @@ -79,12 +79,16 @@ public static AdditionalServlets load(PulsarConfiguration conf) throws IOExcepti return null; } + String[] additionalServletsList = additionalServlets.split(","); + if (additionalServletsList.length == 0) { + return null; + } + AdditionalServletDefinitions definitions = AdditionalServletUtils.searchForServlets(additionalServletDirectory , narExtractionDirectory); ImmutableMap.Builder builder = ImmutableMap.builder(); - String[] additionalServletsList = additionalServlets.split(","); for (String servletName : additionalServletsList) { AdditionalServletMetadata definition = definitions.servlets().get(servletName); if (null == definition) { @@ -106,7 +110,7 @@ public static AdditionalServlets load(PulsarConfiguration conf) throws IOExcepti } Map servlets = builder.build(); - if (servlets != null && !servlets.isEmpty()) { + if (!servlets.isEmpty()) { return new AdditionalServlets(servlets); } diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 6aac8959a1065..c2f808abf7f2b 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - pulsar-broker jar Pulsar Broker - commons-codec commons-codec - org.apache.commons commons-collections4 - org.apache.commons commons-lang3 - org.slf4j slf4j-api - io.netty netty-transport - com.google.protobuf protobuf-java - ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} pulsar-websocket ${project.version} - ${project.groupId} pulsar-client-admin-original ${project.version} - ${project.groupId} pulsar-cli-utils ${project.version} - ${project.groupId} managed-ledger ${project.version} - org.apache.curator curator-recipes - org.apache.bookkeeper stream-storage-server @@ -125,170 +110,139 @@ - org.apache.bookkeeper bookkeeper-tools-framework - ${project.groupId} pulsar-broker-common ${project.version} - ${project.groupId} pulsar-transaction-common ${project.version} - ${project.groupId} pulsar-io-batch-discovery-triggerers ${project.version} test - ${project.groupId} testmocks ${project.version} test - com.github.tomakehurst wiremock-jre8 ${wiremock.version} test - - io.dropwizard.metrics - metrics-core + io.dropwizard.metrics + metrics-core - - org.xerial.snappy - snappy-java + org.xerial.snappy + snappy-java - - ${project.groupId} pulsar-functions-worker ${project.version} - ${project.groupId} pulsar-functions-local-runner-original ${project.version} test - ${project.groupId} pulsar-client-messagecrypto-bc ${project.version} - org.awaitility awaitility test - - org.eclipse.jetty jetty-server - org.eclipse.jetty jetty-alpn-conscrypt-server - org.eclipse.jetty jetty-servlet - org.eclipse.jetty jetty-servlets - org.glassfish.jersey.core jersey-server - org.glassfish.jersey.containers jersey-container-servlet-core - org.glassfish.jersey.containers jersey-container-servlet - org.glassfish.jersey.media jersey-media-json-jackson - org.glassfish.jersey.test-framework jersey-test-framework-core test ${jersey.version} - org.glassfish.jersey.test-framework.providers jersey-test-framework-provider-grizzly2 test ${jersey.version} - jakarta.activation jakarta.activation-api - com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - org.glassfish.jersey.inject jersey-hk2 - com.fasterxml.jackson.module jackson-module-jsonSchema - org.slf4j jcl-over-slf4j - com.google.guava guava - ${project.groupId} pulsar-docs-tools @@ -300,80 +254,65 @@ - com.beust jcommander - io.swagger swagger-annotations provided - io.prometheus simpleclient - io.prometheus simpleclient_jetty - io.prometheus simpleclient_hotspot - io.prometheus simpleclient_caffeine - io.swagger swagger-core provided - org.hdrhistogram HdrHistogram - com.google.code.gson gson - com.github.zafarkhaja java-semver - org.apache.avro avro ${avro.version} - com.carrotsearch hppc - org.roaringbitmap RoaringBitmap - com.github.oshi oshi-core-java11 - ${project.groupId} pulsar-functions-api-examples @@ -381,7 +320,6 @@ pom test - ${project.groupId} pulsar-functions-api-examples-builtin @@ -389,7 +327,6 @@ pom test - ${project.groupId} pulsar-io-batch-data-generator @@ -397,7 +334,6 @@ pom test - ${project.groupId} pulsar-io-data-generator @@ -405,7 +341,6 @@ pom test - ${project.groupId} pulsar-metadata @@ -413,7 +348,6 @@ test-jar test - javax.xml.bind jaxb-api @@ -424,42 +358,33 @@ - com.sun.activation javax.activation - - ${project.groupId} pulsar-transaction-coordinator ${project.version} - - ${project.groupId} pulsar-package-core ${project.version} - io.etcd jetcd-test test - ${project.groupId} pulsar-package-filesystem-storage ${project.version} - - @@ -482,7 +407,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -513,7 +437,6 @@ - maven-dependency-plugin @@ -566,7 +489,6 @@ - org.apache.maven.plugins maven-surefire-plugin @@ -582,7 +504,6 @@ - org.xolstice.maven.plugins protobuf-maven-plugin @@ -652,7 +573,6 @@ - maven-resources-plugin @@ -683,7 +603,6 @@ - @@ -702,7 +621,6 @@ test-jar test - ${project.groupId} pulsar-package-core @@ -710,7 +628,6 @@ test-jar test - ${project.groupId} pulsar-metadata diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index a04a4c137ccbc..83a91e6971e83 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -275,6 +275,7 @@ public class PulsarService implements AutoCloseable, ShutdownService { private TransactionPendingAckStoreProvider transactionPendingAckStoreProvider; private final ExecutorProvider transactionExecutorProvider; private final DefaultMonotonicSnapshotClock monotonicSnapshotClock; + private String brokerId; public enum State { Init, Started, Closing, Closed @@ -307,6 +308,7 @@ public PulsarService(ServiceConfiguration config, // Validate correctness of configuration PulsarConfigurationLoader.isComplete(config); TransactionBatchedWriteValidator.validate(config); + this.config = config; // validate `advertisedAddress`, `advertisedListeners`, `internalListenerName` this.advertisedListeners = MultipleListenerValidator.validateAndAnalysisAdvertisedListener(config); @@ -317,7 +319,6 @@ public PulsarService(ServiceConfiguration config, // use `internalListenerName` listener as `advertisedAddress` this.bindAddress = ServiceConfigurationUtils.getDefaultOrConfiguredAddress(config.getBindAddress()); this.brokerVersion = PulsarVersion.getVersion(); - this.config = config; this.processTerminator = processTerminator; this.loadManagerExecutor = Executors .newSingleThreadScheduledExecutor(new ExecutorProvider.ExtendedThreadFactory("pulsar-load-manager")); @@ -828,6 +829,12 @@ public void start() throws PulsarServerException { this.brokerServiceUrl = brokerUrl(config); this.brokerServiceUrlTls = brokerUrlTls(config); + // the broker id is used in the load manager to identify the broker + // it should not be used for making connections to the broker + this.brokerId = + String.format("%s:%s", advertisedAddress, config.getWebServicePort() + .or(config::getWebServicePortTls).orElseThrow()); + if (this.compactionServiceFactory == null) { this.compactionServiceFactory = loadCompactionServiceFactory(); } @@ -1099,7 +1106,7 @@ private void addWebSocketServiceHandler(WebService webService, } private void handleDeleteCluster(Notification notification) { - if (ClusterResources.pathRepresentsClusterName(notification.getPath()) + if (isRunning() && ClusterResources.pathRepresentsClusterName(notification.getPath()) && notification.getType() == NotificationType.Deleted) { final String clusterName = ClusterResources.clusterNameFromPath(notification.getPath()); getBrokerService().closeAndRemoveReplicationClient(clusterName); @@ -1137,7 +1144,8 @@ protected void startLeaderElectionService() { LOG.info("The load manager extension is enabled. Skipping PulsarService LeaderElectionService."); return; } - this.leaderElectionService = new LeaderElectionService(coordinationService, getSafeWebServiceAddress(), + this.leaderElectionService = + new LeaderElectionService(coordinationService, getBrokerId(), getSafeWebServiceAddress(), state -> { if (state == LeaderElectionState.Leading) { LOG.info("This broker was elected leader"); @@ -1185,7 +1193,7 @@ protected void startLeaderElectionService() { protected void acquireSLANamespace() { try { // Namespace not created hence no need to unload it - NamespaceName nsName = NamespaceService.getSLAMonitorNamespace(getLookupServiceAddress(), config); + NamespaceName nsName = NamespaceService.getSLAMonitorNamespace(getBrokerId(), config); if (!this.pulsarResources.getNamespaceResources().namespaceExists(nsName)) { LOG.info("SLA Namespace = {} doesn't exist.", nsName); return; @@ -1694,10 +1702,16 @@ public String getSafeBrokerServiceUrl() { return brokerServiceUrlTls != null ? brokerServiceUrlTls : brokerServiceUrl; } - public String getLookupServiceAddress() { - return String.format("%s:%s", advertisedAddress, config.getWebServicePortTls().isPresent() - ? config.getWebServicePortTls().get() - : config.getWebServicePort().orElseThrow()); + /** + * Return the broker id. The broker id is used in the load manager to uniquely identify the broker at runtime. + * It should not be used for making connections to the broker. The broker id is available after {@link #start()} + * has been called. + * + * @return broker id + */ + public String getBrokerId() { + return Objects.requireNonNull(brokerId, + "brokerId is not initialized before start has been called"); } public synchronized void addPrometheusRawMetricsProvider(PrometheusRawMetricsProvider metricsProvider) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java index 1526ae18a9057..2ceec18997538 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java @@ -834,6 +834,10 @@ protected static boolean isNotFoundException(Throwable ex) { == Status.NOT_FOUND.getStatusCode(); } + protected static boolean isNot307And404Exception(Throwable ex) { + return !isRedirectException(ex) && !isNotFoundException(ex); + } + protected static String getTopicNotFoundErrorMessage(String topic) { return String.format("Topic %s not found", topic); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java index 3fb1941b33af5..61b354610ac20 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java @@ -26,6 +26,7 @@ import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -34,6 +35,7 @@ import java.util.Objects; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; @@ -80,12 +82,18 @@ public class BrokersBase extends AdminResource { // log a full thread dump when a deadlock is detected in healthcheck once every 10 minutes // to prevent excessive logging private static final long LOG_THREADDUMP_INTERVAL_WHEN_DEADLOCK_DETECTED = 600000L; + // there is a timeout of 60 seconds default in the client(readTimeoutMs), so we need to set the timeout + // a bit shorter than 60 seconds to avoid the client timeout exception thrown before the server timeout exception. + // or we can't propagate the server timeout exception to the client. + private static final Duration HEALTH_CHECK_READ_TIMEOUT = Duration.ofSeconds(58); + private static final TimeoutException HEALTH_CHECK_TIMEOUT_EXCEPTION = + FutureUtil.createTimeoutException("Timeout", BrokersBase.class, "healthCheckRecursiveReadNext(...)"); private volatile long threadDumpLoggedTimestamp; @GET @Path("/{cluster}") @ApiOperation( - value = "Get the list of active brokers (web service addresses) in the cluster." + value = "Get the list of active brokers (broker ids) in the cluster." + "If authorization is not enabled, any cluster name is valid.", response = String.class, responseContainer = "Set") @@ -115,7 +123,7 @@ public void getActiveBrokers(@Suspended final AsyncResponse asyncResponse, @GET @ApiOperation( - value = "Get the list of active brokers (web service addresses) in the local cluster." + value = "Get the list of active brokers (broker ids) in the local cluster." + "If authorization is not enabled", response = String.class, responseContainer = "Set") @@ -141,7 +149,9 @@ public void getLeaderBroker(@Suspended final AsyncResponse asyncResponse) { validateSuperUserAccessAsync().thenAccept(__ -> { LeaderBroker leaderBroker = pulsar().getLeaderElectionService().getCurrentLeader() .orElseThrow(() -> new RestException(Status.NOT_FOUND, "Couldn't find leader broker")); - BrokerInfo brokerInfo = BrokerInfo.builder().serviceUrl(leaderBroker.getServiceUrl()).build(); + BrokerInfo brokerInfo = BrokerInfo.builder() + .serviceUrl(leaderBroker.getServiceUrl()) + .brokerId(leaderBroker.getBrokerId()).build(); LOG.info("[{}] Successfully to get the information of the leader broker.", clientAppId()); asyncResponse.resume(brokerInfo); }) @@ -153,8 +163,8 @@ public void getLeaderBroker(@Suspended final AsyncResponse asyncResponse) { } @GET - @Path("/{clusterName}/{broker-webserviceurl}/ownedNamespaces") - @ApiOperation(value = "Get the list of namespaces served by the specific broker", + @Path("/{clusterName}/{brokerId}/ownedNamespaces") + @ApiOperation(value = "Get the list of namespaces served by the specific broker id", response = NamespaceOwnershipStatus.class, responseContainer = "Map") @ApiResponses(value = { @ApiResponse(code = 307, message = "Current broker doesn't serve the cluster"), @@ -162,9 +172,9 @@ public void getLeaderBroker(@Suspended final AsyncResponse asyncResponse) { @ApiResponse(code = 404, message = "Cluster doesn't exist") }) public void getOwnedNamespaces(@Suspended final AsyncResponse asyncResponse, @PathParam("clusterName") String cluster, - @PathParam("broker-webserviceurl") String broker) { + @PathParam("brokerId") String brokerId) { validateSuperUserAccessAsync() - .thenAccept(__ -> validateBrokerName(broker)) + .thenCompose(__ -> maybeRedirectToBroker(brokerId)) .thenCompose(__ -> validateClusterOwnershipAsync(cluster)) .thenCompose(__ -> pulsar().getNamespaceService().getOwnedNameSpacesStatusAsync()) .thenAccept(asyncResponse::resume) @@ -172,7 +182,7 @@ public void getOwnedNamespaces(@Suspended final AsyncResponse asyncResponse, // If the exception is not redirect exception we need to log it. if (!isRedirectException(ex)) { LOG.error("[{}] Failed to get the namespace ownership status. cluster={}, broker={}", - clientAppId(), cluster, broker); + clientAppId(), cluster, brokerId); } resumeAsyncResponseExceptionally(asyncResponse, ex); return null; @@ -396,10 +406,10 @@ private void checkDeadlockedThreads() { private CompletableFuture internalRunHealthCheck(TopicVersion topicVersion) { - String lookupServiceAddress = pulsar().getLookupServiceAddress(); + String brokerId = pulsar().getBrokerId(); NamespaceName namespaceName = (topicVersion == TopicVersion.V2) - ? NamespaceService.getHeartbeatNamespaceV2(lookupServiceAddress, pulsar().getConfiguration()) - : NamespaceService.getHeartbeatNamespace(lookupServiceAddress, pulsar().getConfiguration()); + ? NamespaceService.getHeartbeatNamespaceV2(brokerId, pulsar().getConfiguration()) + : NamespaceService.getHeartbeatNamespace(brokerId, pulsar().getConfiguration()); final String topicName = String.format("persistent://%s/%s", namespaceName, HEALTH_CHECK_TOPIC_SUFFIX); LOG.info("[{}] Running healthCheck with topic={}", clientAppId(), topicName); final String messageStr = UUID.randomUUID().toString(); @@ -432,7 +442,10 @@ private CompletableFuture internalRunHealthCheck(TopicVersion topicVersion }); throw FutureUtil.wrapToCompletionException(createException); }).thenCompose(reader -> producer.sendAsync(messageStr) - .thenCompose(__ -> healthCheckRecursiveReadNext(reader, messageStr)) + .thenCompose(__ -> FutureUtil.addTimeoutHandling( + healthCheckRecursiveReadNext(reader, messageStr), + HEALTH_CHECK_READ_TIMEOUT, pulsar().getBrokerService().executor(), + () -> HEALTH_CHECK_TIMEOUT_EXCEPTION)) .whenComplete((__, ex) -> { closeAndReCheck(producer, reader, topicOptional.get(), subscriptionName) .whenComplete((unused, innerEx) -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index caaff010439d9..f274cffa46baf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -923,9 +923,9 @@ private CompletableFuture validateLeaderBrokerAsync() { return FutureUtil.failedFuture(new RestException(Response.Status.PRECONDITION_FAILED, errorStr)); } LeaderBroker leaderBroker = pulsar().getLeaderElectionService().getCurrentLeader().get(); - String leaderBrokerUrl = leaderBroker.getServiceUrl(); + String leaderBrokerId = leaderBroker.getBrokerId(); return pulsar().getNamespaceService() - .createLookupResult(leaderBrokerUrl, false, null) + .createLookupResult(leaderBrokerId, false, null) .thenCompose(lookupResult -> { String redirectUrl = isRequestHttps() ? lookupResult.getLookupData().getHttpUrlTls() : lookupResult.getLookupData().getHttpUrl(); @@ -948,7 +948,7 @@ private CompletableFuture validateLeaderBrokerAsync() { return FutureUtil.failedFuture(( new WebApplicationException(Response.temporaryRedirect(redirect).build()))); } catch (MalformedURLException exception) { - log.error("The leader broker url is malformed - {}", leaderBrokerUrl); + log.error("The redirect url is malformed - {}", redirectUrl); return FutureUtil.failedFuture(new RestException(exception)); } }); @@ -984,8 +984,11 @@ public CompletableFuture setNamespaceBundleAffinityAsync(String bundleRang } public CompletableFuture internalUnloadNamespaceBundleAsync(String bundleRange, - String destinationBroker, + String destinationBrokerParam, boolean authoritative) { + String destinationBroker = StringUtils.isBlank(destinationBrokerParam) ? null : + // ensure backward compatibility: strip the possible http:// or https:// prefix + destinationBrokerParam.replaceFirst("http[s]?://", ""); return validateSuperUserAccessAsync() .thenCompose(__ -> setNamespaceBundleAffinityAsync(bundleRange, destinationBroker)) .thenAccept(__ -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 787ffe397de05..0cdb140c7c3d7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -148,6 +148,7 @@ import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.BitSetRecyclable; +import org.apache.pulsar.compaction.Compactor; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; import org.slf4j.Logger; @@ -873,7 +874,7 @@ protected void internalUnloadTopic(AsyncResponse asyncResponse, boolean authorit } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to get partitioned metadata while unloading topic {}", clientAppId(), topicName, ex); } @@ -883,7 +884,7 @@ protected void internalUnloadTopic(AsyncResponse asyncResponse, boolean authorit } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to validate the global namespace ownership while unloading topic {}", clientAppId(), topicName, ex); } @@ -1051,7 +1052,7 @@ private void internalUnloadNonPartitionedTopicAsync(AsyncResponse asyncResponse, })) .exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to unload topic {}, {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -1073,7 +1074,7 @@ private void internalUnloadTransactionCoordinatorAsync(AsyncResponse asyncRespon })) .exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to unload tc {},{}", clientAppId(), topicName.getPartitionIndex(), ex); } @@ -1175,7 +1176,7 @@ protected void internalGetSubscriptions(AsyncResponse asyncResponse, boolean aut } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to get partitioned topic metadata while get" + " subscriptions for topic {}", clientAppId(), topicName, ex); } @@ -1185,7 +1186,7 @@ protected void internalGetSubscriptions(AsyncResponse asyncResponse, boolean aut } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to validate the global namespace/topic ownership while get subscriptions" + " for topic {}", clientAppId(), topicName, ex); } @@ -1194,7 +1195,7 @@ protected void internalGetSubscriptions(AsyncResponse asyncResponse, boolean aut }) ).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to get subscriptions for {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -1233,7 +1234,7 @@ private void internalGetSubscriptionsForNonPartitionedTopic(AsyncResponse asyncR .thenAccept(topic -> asyncResponse.resume(new ArrayList<>(topic.getSubscriptions().keys()))) .exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to get list of subscriptions for {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -1342,7 +1343,7 @@ protected void internalGetManagedLedgerInfo(AsyncResponse asyncResponse, boolean } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to get partitioned metadata while get managed info for {}", clientAppId(), topicName, ex); } @@ -1352,7 +1353,7 @@ protected void internalGetManagedLedgerInfo(AsyncResponse asyncResponse, boolean } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to validate the global namespace ownership while get managed info for {}", clientAppId(), topicName, ex); } @@ -1471,7 +1472,7 @@ protected void internalGetPartitionedStats(AsyncResponse asyncResponse, boolean }); }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to get partitioned internal stats for {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -1526,7 +1527,7 @@ protected void internalGetPartitionedStatsInternal(AsyncResponse asyncResponse, }); }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to get partitioned internal stats for {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -1654,7 +1655,7 @@ private void internalAnalyzeSubscriptionBacklogForNonPartitionedTopic(AsyncRespo }).exceptionally(ex -> { Throwable cause = ex.getCause(); // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to analyze subscription backlog {} {}", clientAppId(), topicName, subName, cause); } @@ -1681,7 +1682,7 @@ private void internalUpdateSubscriptionPropertiesForNonPartitionedTopic(AsyncRes }).exceptionally(ex -> { Throwable cause = ex.getCause(); // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to update subscription {} {}", clientAppId(), topicName, subName, cause); } asyncResponse.resume(new RestException(cause)); @@ -1710,7 +1711,7 @@ private void internalGetSubscriptionPropertiesForNonPartitionedTopic(AsyncRespon }).exceptionally(ex -> { Throwable cause = ex.getCause(); // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to update subscription {} {}", clientAppId(), topicName, subName, cause); } asyncResponse.resume(new RestException(cause)); @@ -1879,7 +1880,7 @@ protected void internalSkipAllMessages(AsyncResponse asyncResponse, String subNa } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to skip all messages for subscription {} on topic {}", clientAppId(), subName, topicName, ex); } @@ -1923,7 +1924,7 @@ private CompletableFuture internalSkipAllMessagesForNonPartitionedTopicAsy } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to skip all messages for subscription {} on topic {}", clientAppId(), subName, topicName, ex); } @@ -1987,7 +1988,7 @@ protected void internalSkipMessages(AsyncResponse asyncResponse, String subName, }) ).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to skip {} messages {} {}", clientAppId(), numMessages, topicName, subName, ex); } @@ -2057,7 +2058,7 @@ protected void internalExpireMessagesForAllSubscriptions(AsyncResponse asyncResp ) ).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to expire messages for all subscription on topic {}", clientAppId(), topicName, ex); } @@ -2090,10 +2091,9 @@ private void internalExpireMessagesForAllSubscriptionsForNonPartitionedTopic(Asy final List> futures = new ArrayList<>((int) topic.getReplicators().size()); List subNames = - new ArrayList<>((int) topic.getReplicators().size() - + (int) topic.getSubscriptions().size()); - subNames.addAll(topic.getReplicators().keys()); - subNames.addAll(topic.getSubscriptions().keys()); + new ArrayList<>((int) topic.getSubscriptions().size()); + subNames.addAll(topic.getSubscriptions().keys().stream().filter( + subName -> !subName.equals(Compactor.COMPACTION_SUBSCRIPTION)).toList()); for (int i = 0; i < subNames.size(); i++) { try { futures.add(internalExpireMessagesByTimestampForSinglePartitionAsync(partitionMetadata, @@ -2125,7 +2125,7 @@ private void internalExpireMessagesForAllSubscriptionsForNonPartitionedTopic(Asy }) ).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to expire messages for all subscription up to {} on {}", clientAppId(), expireTimeInSeconds, topicName, ex); } @@ -2332,7 +2332,7 @@ protected void internalCreateSubscription(AsyncResponse asyncResponse, String su })).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to create subscription {} on topic {}", clientAppId(), subscriptionName, topicName, ex); } @@ -2342,7 +2342,7 @@ protected void internalCreateSubscription(AsyncResponse asyncResponse, String su } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to create subscription {} on topic {}", clientAppId(), subscriptionName, topicName, ex); } @@ -2473,7 +2473,7 @@ protected void internalUpdateSubscriptionProperties(AsyncResponse asyncResponse, } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to update subscription {} from topic {}", clientAppId(), subName, topicName, ex); } @@ -2513,7 +2513,7 @@ protected void internalAnalyzeSubscriptionBacklog(AsyncResponse asyncResponse, S }) .exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to analyze back log of subscription {} from topic {}", clientAppId(), subName, topicName, ex); } @@ -2598,7 +2598,7 @@ protected void internalGetSubscriptionProperties(AsyncResponse asyncResponse, St } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to update subscription {} from topic {}", clientAppId(), subName, topicName, ex); } @@ -2684,7 +2684,7 @@ protected void internalResetCursorOnPosition(AsyncResponse asyncResponse, String }); }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.warn("[{}][{}] Failed to reset cursor on subscription {} to position {}", clientAppId(), topicName, subName, messageId, ex.getCause()); } @@ -2693,7 +2693,7 @@ protected void internalResetCursorOnPosition(AsyncResponse asyncResponse, String }); }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.warn("[{}][{}] Failed to reset cursor on subscription {} to position {}", clientAppId(), topicName, subName, messageId, ex.getCause()); } @@ -2826,6 +2826,9 @@ protected CompletableFuture internalGetMessageById(long ledgerId, long @Override public void readEntryFailed(ManagedLedgerException exception, Object ctx) { + if (exception instanceof ManagedLedgerException.LedgerNotExistException) { + throw new RestException(Status.NOT_FOUND, "Message id not found"); + } throw new RestException(exception); } @@ -2882,28 +2885,62 @@ protected CompletableFuture internalGetMessageIdByTimestampAsync(long throw new RestException(Status.METHOD_NOT_ALLOWED, "Get message ID by timestamp on a non-persistent topic is not allowed"); } - ManagedLedger ledger = ((PersistentTopic) topic).getManagedLedger(); - return ledger.asyncFindPosition(entry -> { + final PersistentTopic persistentTopic = (PersistentTopic) topic; + + return persistentTopic.getTopicCompactionService().readLastCompactedEntry().thenCompose(lastEntry -> { + if (lastEntry == null) { + return findMessageIdByPublishTime(timestamp, persistentTopic.getManagedLedger()); + } + MessageMetadata metadata; + Position position = lastEntry.getPosition(); try { - long entryTimestamp = Commands.getEntryTimestamp(entry.getDataBuffer()); - return MessageImpl.isEntryPublishedEarlierThan(entryTimestamp, timestamp); - } catch (Exception e) { - log.error("[{}] Error deserializing message for message position find", topicName, e); + metadata = Commands.parseMessageMetadata(lastEntry.getDataBuffer()); } finally { - entry.release(); + lastEntry.release(); } - return false; - }).thenApply(position -> { - if (position == null) { - return null; + if (timestamp == metadata.getPublishTime()) { + return CompletableFuture.completedFuture(new MessageIdImpl(position.getLedgerId(), + position.getEntryId(), topicName.getPartitionIndex())); + } else if (timestamp < metadata.getPublishTime()) { + return persistentTopic.getTopicCompactionService().findEntryByPublishTime(timestamp) + .thenApply(compactedEntry -> { + try { + return new MessageIdImpl(compactedEntry.getLedgerId(), + compactedEntry.getEntryId(), topicName.getPartitionIndex()); + } finally { + compactedEntry.release(); + } + }); } else { - return new MessageIdImpl(position.getLedgerId(), position.getEntryId(), - topicName.getPartitionIndex()); + return findMessageIdByPublishTime(timestamp, persistentTopic.getManagedLedger()); } }); }); } + private CompletableFuture findMessageIdByPublishTime(long timestamp, ManagedLedger managedLedger) { + return managedLedger.asyncFindPosition(entry -> { + try { + long entryTimestamp = Commands.getEntryTimestamp(entry.getDataBuffer()); + return MessageImpl.isEntryPublishedEarlierThan(entryTimestamp, timestamp); + } catch (Exception e) { + log.error("[{}] Error deserializing message for message position find", + topicName, + e); + } finally { + entry.release(); + } + return false; + }).thenApply(position -> { + if (position == null) { + return null; + } else { + return new MessageIdImpl(position.getLedgerId(), position.getEntryId(), + topicName.getPartitionIndex()); + } + }); + } + protected CompletableFuture internalPeekNthMessageAsync(String subName, int messagePosition, boolean authoritative) { CompletableFuture ret; @@ -3292,7 +3329,7 @@ protected void internalGetBacklogSizeByMessageId(AsyncResponse asyncResponse, } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to get backlog size for topic {}", clientAppId(), topicName, ex); } @@ -3300,7 +3337,7 @@ protected void internalGetBacklogSizeByMessageId(AsyncResponse asyncResponse, return null; })).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to validate global namespace ownership " + "to get backlog size for topic {}", clientAppId(), topicName, ex); } @@ -3838,7 +3875,7 @@ protected void internalTerminatePartitionedTopic(AsyncResponse asyncResponse, bo } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to terminate topic {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -3846,7 +3883,7 @@ protected void internalTerminatePartitionedTopic(AsyncResponse asyncResponse, bo }) ).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to terminate topic {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -3936,7 +3973,7 @@ protected void internalExpireMessagesByTimestamp(AsyncResponse asyncResponse, St ).exceptionally(ex -> { Throwable cause = FutureUtil.unwrapCompletionException(ex); // If the exception is not redirect exception we need to log it. - if (!isRedirectException(cause)) { + if (!isNot307And404Exception(cause)) { if (cause instanceof RestException) { log.warn("[{}] Failed to expire messages up to {} on {}: {}", clientAppId(), expireTimeInSeconds, topicName, cause.toString()); @@ -4051,7 +4088,7 @@ protected void internalExpireMessagesByPosition(AsyncResponse asyncResponse, Str messageId, isExcluded, batchIndex); }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to expire messages up to {} on subscription {} to position {}", clientAppId(), topicName, subName, messageId, ex); } @@ -4201,7 +4238,7 @@ protected void internalTriggerCompaction(AsyncResponse asyncResponse, boolean au } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to trigger compaction on topic {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -4210,7 +4247,7 @@ protected void internalTriggerCompaction(AsyncResponse asyncResponse, boolean au } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to validate global namespace ownership to trigger compaction on topic {}", clientAppId(), topicName, ex); } @@ -4239,7 +4276,7 @@ protected void internalTriggerCompactionNonPartitionedTopic(AsyncResponse asyncR } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to trigger compaction for {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -4275,7 +4312,7 @@ protected void internalTriggerOffload(AsyncResponse asyncResponse, } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to trigger offload for {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -4292,7 +4329,7 @@ protected void internalOffloadStatus(AsyncResponse asyncResponse, boolean author asyncResponse.resume(offloadProcessStatus); }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to offload status on topic {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -4569,7 +4606,7 @@ protected void internalGetLastMessageId(AsyncResponse asyncResponse, boolean aut }); }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to get last messageId {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -4947,9 +4984,7 @@ protected CompletableFuture internalRemoveSubscribeRate(boolean isGlobal) protected void handleTopicPolicyException(String methodName, Throwable thr, AsyncResponse asyncResponse) { Throwable cause = thr.getCause(); - if (!(cause instanceof WebApplicationException) || !( - ((WebApplicationException) cause).getResponse().getStatus() == 307 - || ((WebApplicationException) cause).getResponse().getStatus() == 404)) { + if (isNot307And404Exception(cause)) { log.error("[{}] Failed to perform {} on topic {}", clientAppId(), methodName, topicName, cause); } @@ -5075,7 +5110,7 @@ protected void internalSetReplicatedSubscriptionStatus(AsyncResponse asyncRespon resultFuture.exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.warn("[{}] Failed to change replicated subscription status to {} - {} {}", clientAppId(), enabled, topicName, subName, ex); } @@ -5122,7 +5157,7 @@ private void internalSetReplicatedSubscriptionStatusForNonPartitionedTopic( } ).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to set replicated subscription status on {} {}", clientAppId(), topicName, subName, ex); } @@ -5223,7 +5258,7 @@ protected void internalGetReplicatedSubscriptionStatus(AsyncResponse asyncRespon } resultFuture.exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to get replicated subscription status on {} {}", clientAppId(), topicName, subName, ex); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java index 454b8f0fac82c..286366c8b5834 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java @@ -238,7 +238,7 @@ private CompletableFuture validateOwnershipAndOperationAsync(boolean autho protected boolean shouldPrintErrorLog(Throwable ex) { - return !isRedirectException(ex) && !isNotFoundException(ex); + return isNot307And404Exception(ex); } private static final Logger log = LoggerFactory.getLogger(SchemasResourceBase.class); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java index d4795393f9b03..a22ad4b242f57 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java @@ -132,7 +132,7 @@ public void getInternalStats( }) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to get internal stats for topic {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -479,7 +479,7 @@ public void getListFromBundle( } asyncResponse.resume(topicList); }).exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to list topics on namespace bundle {}/{}", clientAppId(), namespaceName, bundleRange, ex); } @@ -488,7 +488,7 @@ public void getListFromBundle( }); } }).exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to list topics on namespace bundle {}/{}", clientAppId(), namespaceName, bundleRange, ex); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java index 9ccbc0ecba171..9094a4642a0cb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java @@ -119,7 +119,7 @@ public void getList( internalGetListAsync(Optional.ofNullable(bundle)) .thenAccept(topicList -> asyncResponse.resume(filterSystemTopic(topicList, includeSystemTopic))) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get topic list {}", clientAppId(), namespaceName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -150,7 +150,7 @@ public void getPartitionedTopicList( .thenAccept(partitionedTopicList -> asyncResponse.resume( filterSystemTopic(partitionedTopicList, includeSystemTopic))) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get partitioned topic list {}", clientAppId(), namespaceName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -335,7 +335,7 @@ public void createNonPartitionedTopic( internalCreateNonPartitionedTopicAsync(authoritative, properties) .thenAccept(__ -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to create non-partitioned topic {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -825,7 +825,7 @@ public void updatePartitionedTopic( asyncResponse.resume(Response.noContent().build()); }) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}][{}] Failed to update partition to {}", clientAppId(), topicName, numPartitions, ex); } @@ -934,7 +934,7 @@ public void getProperties( internalGetPropertiesAsync(authoritative) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get topic {} properties", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -970,7 +970,7 @@ public void updateProperties( internalUpdatePropertiesAsync(authoritative, properties) .thenAccept(__ -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to update topic {} properties", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -1004,7 +1004,7 @@ public void removeProperties( internalRemovePropertiesAsync(authoritative, key) .thenAccept(__ -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to remove key {} in properties on topic {}", clientAppId(), key, topicName, ex); } @@ -1125,7 +1125,7 @@ public void deleteTopic( } else if (isManagedLedgerNotFoundException(t)) { ex = new RestException(Response.Status.NOT_FOUND, getTopicNotFoundErrorMessage(topicName.toString())); - } else if (!isRedirectException(ex)) { + } else if (isNot307And404Exception(ex)) { log.error("[{}] Failed to delete topic {}", clientAppId(), topicName, t); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -1209,7 +1209,7 @@ public void getStats( .thenAccept(asyncResponse::resume) .exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get stats for {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -1243,7 +1243,7 @@ public void getInternalStats( internalGetInternalStatsAsync(authoritative, metadata) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get internal stats for topic {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -1892,7 +1892,7 @@ public void peekNthMessage( internalPeekNthMessageAsync(decode(encodedSubName), messagePosition, authoritative) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get peek nth message for topic {} subscription {}", clientAppId(), topicName, decode(encodedSubName), ex); } @@ -1934,7 +1934,7 @@ public void examineMessage( internalExamineMessageAsync(initialPosition, messagePosition, authoritative) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to examine a specific message on the topic {}", clientAppId(), topicName, ex); } @@ -1976,7 +1976,7 @@ public void getMessageById( .thenAccept(asyncResponse::resume) .exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get message with ledgerId {} entryId {} from {}", clientAppId(), ledgerId, entryId, topicName, ex); } @@ -2020,7 +2020,7 @@ public void getMessageIdByTimestamp( } }) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get message ID by timestamp {} from {}", clientAppId(), timestamp, topicName, ex); } @@ -2055,7 +2055,7 @@ public void getBacklog( log.warn("[{}] Failed to get topic backlog {}: Namespace does not exist", clientAppId(), namespaceName); ex = new RestException(Response.Status.NOT_FOUND, "Namespace does not exist"); - } else if (!isRedirectException(ex)) { + } else if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get estimated backlog for topic {}", clientAppId(), encodedTopic, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -2435,7 +2435,8 @@ public void getRetention(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RETENTION, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetRetention(applied, isGlobal)) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { @@ -2462,7 +2463,8 @@ public void setRetention(@Suspended final AsyncResponse asyncResponse, @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal, @ApiParam(value = "Retention policies for the specified topic") RetentionPolicies retention) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RETENTION, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetRetention(retention, isGlobal)) .thenRun(() -> { try { @@ -2498,7 +2500,8 @@ public void removeRetention(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RETENTION, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalRemoveRetention(isGlobal)) .thenRun(() -> { log.info("[{}] Successfully remove retention: namespace={}, topic={}", @@ -3092,7 +3095,7 @@ public void terminate( internalTerminateAsync(authoritative) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to terminated topic {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -3188,7 +3191,7 @@ public void compactionStatus( internalCompactionStatusAsync(authoritative) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get the status of a compaction operation for the topic {}", clientAppId(), topicName, ex); } @@ -3327,7 +3330,7 @@ public void trimTopic( validateTopicName(tenant, namespace, encodedTopic); internalTrimTopic(asyncResponse, authoritative).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to trim topic {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Transactions.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Transactions.java index 1bdc2255085f1..667d8ce581ece 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Transactions.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Transactions.java @@ -105,7 +105,7 @@ public void getTransactionInBufferStats(@Suspended final AsyncResponse asyncResp Long.parseLong(leastSigBits)) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to get transaction state in transaction buffer {}", clientAppId(), topicName, ex); } @@ -143,7 +143,7 @@ public void getTransactionInPendingAckStats(@Suspended final AsyncResponse async Long.parseLong(leastSigBits), subName) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to get transaction state in pending ack {}", clientAppId(), topicName, ex); } @@ -181,7 +181,7 @@ public void getTransactionBufferStats(@Suspended final AsyncResponse asyncRespon internalGetTransactionBufferStats(authoritative, lowWaterMarks, segmentStats) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to get transaction buffer stats in topic {}", clientAppId(), topicName, ex); } @@ -217,7 +217,7 @@ public void getPendingAckStats(@Suspended final AsyncResponse asyncResponse, internalGetPendingAckStats(authoritative, subName, lowWaterMarks) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to get transaction pending ack stats in topic {}", clientAppId(), topicName, ex); } @@ -314,7 +314,7 @@ public void getPendingAckInternalStats(@Suspended final AsyncResponse asyncRespo internalGetPendingAckInternalStats(authoritative, subName, metadata) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to get pending ack internal stats {}", clientAppId(), topicName, ex); } @@ -365,7 +365,7 @@ public void getTransactionBufferInternalStats(@Suspended final AsyncResponse asy internalGetTransactionBufferInternalStats(authoritative, metadata) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (!isNot307And404Exception(ex)) { log.error("[{}] Failed to get transaction buffer internal stats {}", clientAppId(), topicName, ex); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java index cef3f0eb609a1..30d1874a97299 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java @@ -59,6 +59,9 @@ public BrokerInterceptors(Map intercep * @return the collection of broker event interceptor */ public static BrokerInterceptor load(ServiceConfiguration conf) throws IOException { + if (conf.getBrokerInterceptors().isEmpty()) { + return null; + } BrokerInterceptorDefinitions definitions = BrokerInterceptorUtils.searchForInterceptors(conf.getBrokerInterceptorsDirectory(), conf.getNarExtractionDirectory()); @@ -87,7 +90,7 @@ public static BrokerInterceptor load(ServiceConfiguration conf) throws IOExcepti }); Map interceptors = builder.build(); - if (interceptors != null && !interceptors.isEmpty()) { + if (!interceptors.isEmpty()) { return new BrokerInterceptors(interceptors); } else { return null; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderBroker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderBroker.java index acd34e151ed2a..d7c21de5ea1fa 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderBroker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderBroker.java @@ -30,5 +30,23 @@ @AllArgsConstructor @NoArgsConstructor public class LeaderBroker { + private String brokerId; private String serviceUrl; + + public String getBrokerId() { + if (brokerId != null) { + return brokerId; + } else { + // for backward compatibility at runtime with older versions of Pulsar + return parseHostAndPort(serviceUrl); + } + } + + private static String parseHostAndPort(String serviceUrl) { + int uriSeparatorPos = serviceUrl.indexOf("://"); + if (uriSeparatorPos == -1) { + throw new IllegalArgumentException("'" + serviceUrl + "' isn't an URI."); + } + return serviceUrl.substring(uriSeparatorPos + 3); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java index 05fe4353f3e76..2e53b54e98f61 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java @@ -35,17 +35,17 @@ public class LeaderElectionService implements AutoCloseable { private final LeaderElection leaderElection; private final LeaderBroker localValue; - public LeaderElectionService(CoordinationService cs, String localWebServiceAddress, - Consumer listener) { - this(cs, localWebServiceAddress, ELECTION_ROOT, listener); + public LeaderElectionService(CoordinationService cs, String brokerId, + String serviceUrl, Consumer listener) { + this(cs, brokerId, serviceUrl, ELECTION_ROOT, listener); } public LeaderElectionService(CoordinationService cs, - String localWebServiceAddress, - String electionRoot, + String brokerId, + String serviceUrl, String electionRoot, Consumer listener) { this.leaderElection = cs.getLeaderElection(LeaderBroker.class, electionRoot, listener); - this.localValue = new LeaderBroker(localWebServiceAddress); + this.localValue = new LeaderBroker(brokerId, serviceUrl); } public void start() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java index 80f887d394dd9..f9f36b705d4c4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java @@ -43,7 +43,7 @@ public class NoopLoadManager implements LoadManager { private PulsarService pulsar; - private String lookupServiceAddress; + private String brokerId; private ResourceUnit localResourceUnit; private LockManager lockManager; private Map bundleBrokerAffinityMap; @@ -57,16 +57,15 @@ public void initialize(PulsarService pulsar) { @Override public void start() throws PulsarServerException { - lookupServiceAddress = pulsar.getLookupServiceAddress(); - localResourceUnit = new SimpleResourceUnit(String.format("http://%s", lookupServiceAddress), - new PulsarResourceDescription()); + brokerId = pulsar.getBrokerId(); + localResourceUnit = new SimpleResourceUnit(brokerId, new PulsarResourceDescription()); LocalBrokerData localData = new LocalBrokerData(pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls(), pulsar.getAdvertisedListeners()); localData.setProtocols(pulsar.getProtocolDataToAdvertise()); localData.setLoadManagerClassName(this.pulsar.getConfig().getLoadManagerClassName()); - String brokerReportPath = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + lookupServiceAddress; + String brokerReportPath = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + brokerId; try { log.info("Acquiring broker resource lock on {}", brokerReportPath); @@ -129,12 +128,12 @@ public void disableBroker() throws Exception { @Override public Set getAvailableBrokers() throws Exception { - return Collections.singleton(lookupServiceAddress); + return Collections.singleton(brokerId); } @Override public CompletableFuture> getAvailableBrokersAsync() { - return CompletableFuture.completedFuture(Collections.singleton(lookupServiceAddress)); + return CompletableFuture.completedFuture(Collections.singleton(brokerId)); } @Override @@ -153,7 +152,6 @@ public String setNamespaceBundleAffinity(String bundle, String broker) { if (StringUtils.isBlank(broker)) { return this.bundleBrokerAffinityMap.remove(bundle); } - broker = broker.replaceFirst("http[s]?://", ""); return this.bundleBrokerAffinityMap.put(bundle, broker); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ResourceUnit.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ResourceUnit.java index ef4dd2a97b280..c28a8be4c0d3a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ResourceUnit.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ResourceUnit.java @@ -23,8 +23,6 @@ */ public interface ResourceUnit extends Comparable { - String PROPERTY_KEY_BROKER_ZNODE_NAME = "__advertised_addr"; - String getResourceId(); ResourceDescription getAvailableResource(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java index bfdaa078f1999..18e30ddf922d0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java @@ -82,7 +82,7 @@ public BrokerRegistryImpl(PulsarService pulsar) { this.brokerLookupDataLockManager = pulsar.getCoordinationService().getLockManager(BrokerLookupData.class); this.scheduler = pulsar.getLoadManagerExecutor(); this.listeners = new ArrayList<>(); - this.brokerId = pulsar.getLookupServiceAddress(); + this.brokerId = pulsar.getBrokerId(); this.brokerLookupData = new BrokerLookupData( pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 581183cf95ad3..4d642e400793a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -21,6 +21,7 @@ import static java.lang.String.format; import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Follower; import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Leader; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.TOPIC; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Success; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Admin; import static org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared.getNamespaceBundle; @@ -118,6 +119,8 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { private static final long MONITOR_INTERVAL_IN_MILLIS = 120_000; + public static final long COMPACTION_THRESHOLD = 5 * 1024 * 1024; + private static final String ELECTION_ROOT = "/loadbalance/extension/leader"; private PulsarService pulsar; @@ -174,6 +177,8 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { private volatile boolean started = false; + private boolean configuredSystemTopics = false; + private final AssignCounter assignCounter = new AssignCounter(); @Getter private final UnloadCounter unloadCounter = new UnloadCounter(); @@ -268,7 +273,7 @@ public static boolean isLoadManagerExtensionEnabled(ServiceConfiguration conf) { } public static boolean isLoadManagerExtensionEnabled(PulsarService pulsar) { - return pulsar.getLoadManager().get() instanceof ExtensibleLoadManagerImpl; + return pulsar.getLoadManager().get() instanceof ExtensibleLoadManagerWrapper; } public static ExtensibleLoadManagerImpl get(LoadManager loadManager) { @@ -309,6 +314,27 @@ private static void createSystemTopics(PulsarService pulsar) throws PulsarServer createSystemTopic(pulsar, TOP_BUNDLES_LOAD_DATA_STORE_TOPIC); } + private static boolean configureSystemTopics(PulsarService pulsar) { + try { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar) + && (pulsar.getConfiguration().isSystemTopicEnabled()) + && pulsar.getConfiguration().isTopicLevelPoliciesEnabled()) { + Long threshold = pulsar.getAdminClient().topicPolicies().getCompactionThreshold(TOPIC); + if (threshold == null || COMPACTION_THRESHOLD != threshold.longValue()) { + pulsar.getAdminClient().topicPolicies().setCompactionThreshold(TOPIC, COMPACTION_THRESHOLD); + log.info("Set compaction threshold: {} bytes for system topic {}.", COMPACTION_THRESHOLD, TOPIC); + } + } else { + log.warn("System topic or topic level policies is disabled. " + + "{} compaction threshold follows the broker or namespace policies.", TOPIC); + } + return true; + } catch (Exception e) { + log.error("Failed to set compaction threshold for system topic:{}", TOPIC, e); + } + return false; + } + /** * Gets the assigned broker for the given topic. * @param pulsar PulsarService instance @@ -348,7 +374,8 @@ public void start() throws PulsarServerException { try { this.brokerRegistry = new BrokerRegistryImpl(pulsar); this.leaderElectionService = new LeaderElectionService( - pulsar.getCoordinationService(), pulsar.getSafeWebServiceAddress(), ELECTION_ROOT, + pulsar.getCoordinationService(), pulsar.getBrokerId(), + pulsar.getSafeWebServiceAddress(), ELECTION_ROOT, state -> { pulsar.getLoadManagerExecutor().execute(() -> { if (state == LeaderElectionState.Leading) { @@ -377,9 +404,9 @@ public void start() throws PulsarServerException { try { this.brokerLoadDataStore = LoadDataStoreFactory - .create(pulsar.getClient(), BROKER_LOAD_DATA_STORE_TOPIC, BrokerLoadData.class); + .create(pulsar, BROKER_LOAD_DATA_STORE_TOPIC, BrokerLoadData.class); this.topBundlesLoadDataStore = LoadDataStoreFactory - .create(pulsar.getClient(), TOP_BUNDLES_LOAD_DATA_STORE_TOPIC, TopBundlesLoadData.class); + .create(pulsar, TOP_BUNDLES_LOAD_DATA_STORE_TOPIC, TopBundlesLoadData.class); } catch (LoadDataStoreException e) { throw new PulsarServerException(e); } @@ -437,7 +464,10 @@ public void start() throws PulsarServerException { this.splitScheduler.start(); this.initWaiter.countDown(); this.started = true; + log.info("Started load manager."); } catch (Exception ex) { + log.error("Failed to start the extensible load balance and close broker registry {}.", + this.brokerRegistry, ex); if (this.brokerRegistry != null) { brokerRegistry.close(); } @@ -569,7 +599,7 @@ private CompletableFuture> dedupeLookupRequest( if (ex != null) { assignCounter.incrementFailure(); } - lookupRequests.remove(key, newFutureCreated.getValue()); + lookupRequests.remove(key); }); } } @@ -782,18 +812,23 @@ public void close() throws PulsarServerException { } public static boolean isInternalTopic(String topic) { - return topic.startsWith(ServiceUnitStateChannelImpl.TOPIC) + return topic.startsWith(TOPIC) || topic.startsWith(BROKER_LOAD_DATA_STORE_TOPIC) || topic.startsWith(TOP_BUNDLES_LOAD_DATA_STORE_TOPIC); } @VisibleForTesting - void playLeader() { + synchronized void playLeader() { log.info("This broker:{} is setting the role from {} to {}", - pulsar.getLookupServiceAddress(), role, Leader); + pulsar.getBrokerId(), role, Leader); int retry = 0; + boolean becameFollower = false; while (!Thread.currentThread().isInterrupted()) { try { + if (!serviceUnitStateChannel.isChannelOwner()) { + becameFollower = true; + break; + } initWaiter.await(); // Confirm the system topics have been created or create them if they do not exist. // If the leader has changed, the new leader need to reset @@ -806,8 +841,8 @@ void playLeader() { serviceUnitStateChannel.scheduleOwnershipMonitor(); break; } catch (Throwable e) { - log.error("The broker:{} failed to set the role. Retrying {} th ...", - pulsar.getLookupServiceAddress(), ++retry, e); + log.warn("The broker:{} failed to set the role. Retrying {} th ...", + pulsar.getBrokerId(), ++retry, e); try { Thread.sleep(Math.min(retry * 10, MAX_ROLE_CHANGE_RETRY_DELAY_IN_MILLIS)); } catch (InterruptedException ex) { @@ -817,8 +852,15 @@ void playLeader() { } } } + + if (becameFollower) { + log.warn("The broker:{} became follower while initializing leader role.", pulsar.getBrokerId()); + playFollower(); + return; + } + role = Leader; - log.info("This broker:{} plays the leader now.", pulsar.getLookupServiceAddress()); + log.info("This broker:{} plays the leader now.", pulsar.getBrokerId()); // flush the load data when the leader is elected. brokerLoadDataReporter.reportAsync(true); @@ -826,12 +868,17 @@ void playLeader() { } @VisibleForTesting - void playFollower() { + synchronized void playFollower() { log.info("This broker:{} is setting the role from {} to {}", - pulsar.getLookupServiceAddress(), role, Follower); + pulsar.getBrokerId(), role, Follower); int retry = 0; + boolean becameLeader = false; while (!Thread.currentThread().isInterrupted()) { try { + if (serviceUnitStateChannel.isChannelOwner()) { + becameLeader = true; + break; + } initWaiter.await(); unloadScheduler.close(); serviceUnitStateChannel.cancelOwnershipMonitor(); @@ -840,8 +887,8 @@ void playFollower() { topBundlesLoadDataStore.startProducer(); break; } catch (Throwable e) { - log.error("The broker:{} failed to set the role. Retrying {} th ...", - pulsar.getLookupServiceAddress(), ++retry, e); + log.warn("The broker:{} failed to set the role. Retrying {} th ...", + pulsar.getBrokerId(), ++retry, e); try { Thread.sleep(Math.min(retry * 10, MAX_ROLE_CHANGE_RETRY_DELAY_IN_MILLIS)); } catch (InterruptedException ex) { @@ -851,8 +898,15 @@ void playFollower() { } } } + + if (becameLeader) { + log.warn("This broker:{} became leader while initializing follower role.", pulsar.getBrokerId()); + playLeader(); + return; + } + role = Follower; - log.info("This broker:{} plays a follower now.", pulsar.getLookupServiceAddress()); + log.info("This broker:{} plays a follower now.", pulsar.getBrokerId()); // flush the load data when the leader is elected. brokerLoadDataReporter.reportAsync(true); @@ -880,7 +934,8 @@ public List getMetrics() { return metricsCollection; } - private void monitor() { + @VisibleForTesting + protected void monitor() { try { initWaiter.await(); @@ -888,6 +943,11 @@ private void monitor() { // Periodically check the role in case ZK watcher fails. var isChannelOwner = serviceUnitStateChannel.isChannelOwner(); if (isChannelOwner) { + // System topic config might fail due to the race condition + // with topic policy init(Topic policies cache have not init). + if (!configuredSystemTopics) { + configuredSystemTopics = configureSystemTopics(pulsar); + } if (role != Leader) { log.warn("Current role:{} does not match with the channel ownership:{}. " + "Playing the leader role.", role, isChannelOwner); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java index 18e949537dedb..cd1561cb70e2d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java @@ -118,13 +118,11 @@ public void setLoadReportForceUpdateFlag() { @Override public void writeLoadReportOnZookeeper() throws Exception { // No-op, this operation is not useful, the load data reporter will automatically write. - throw new UnsupportedOperationException(); } @Override public void writeResourceQuotasToZooKeeper() throws Exception { // No-op, this operation is not useful, the load data reporter will automatically write. - throw new UnsupportedOperationException(); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 713d98b72507e..d8aacb86902a1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -73,6 +73,7 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.LeaderBroker; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; @@ -124,7 +125,7 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel { private final ServiceConfiguration config; private final Schema schema; private final Map> getOwnerRequests; - private final String lookupServiceAddress; + private final String brokerId; private final Map> cleanupJobs; private final StateChangeListeners stateChangeListeners; private ExtensibleLoadManagerImpl loadManager; @@ -201,7 +202,7 @@ enum MetadataState { public ServiceUnitStateChannelImpl(PulsarService pulsar) { this.pulsar = pulsar; this.config = pulsar.getConfig(); - this.lookupServiceAddress = pulsar.getLookupServiceAddress(); + this.brokerId = pulsar.getBrokerId(); this.schema = Schema.JSON(ServiceUnitStateData.class); this.getOwnerRequests = new ConcurrentHashMap<>(); this.cleanupJobs = new ConcurrentHashMap<>(); @@ -249,7 +250,7 @@ public void scheduleOwnershipMonitor() { }, 0, ownershipMonitorDelayTimeInSecs, SECONDS); log.info("This leader broker:{} started the ownership monitor.", - lookupServiceAddress); + brokerId); } } @@ -258,13 +259,13 @@ public void cancelOwnershipMonitor() { monitorTask.cancel(false); monitorTask = null; log.info("This previous leader broker:{} stopped the ownership monitor.", - lookupServiceAddress); + brokerId); } } @Override public void cleanOwnerships() { - doCleanup(lookupServiceAddress); + doCleanup(brokerId); } public synchronized void start() throws PulsarServerException { @@ -430,19 +431,8 @@ public CompletableFuture> getChannelOwnerAsync() { new IllegalStateException("Invalid channel state:" + channelState.name())); } - return leaderElectionService.readCurrentLeader().thenApply(leader -> { - //expecting http://broker-xyz:port - // TODO: discard this protocol prefix removal - // by a util func that returns lookupServiceAddress(serviceUrl) - if (leader.isPresent()) { - String broker = leader.get().getServiceUrl(); - broker = broker.substring(broker.lastIndexOf('/') + 1); - return Optional.of(broker); - } else { - return Optional.empty(); - } - } - ); + return leaderElectionService.readCurrentLeader() + .thenApply(leader -> leader.map(LeaderBroker::getBrokerId)); } public CompletableFuture isChannelOwnerAsync() { @@ -484,7 +474,35 @@ public boolean isOwner(String serviceUnit, String targetBroker) { } public boolean isOwner(String serviceUnit) { - return isOwner(serviceUnit, lookupServiceAddress); + return isOwner(serviceUnit, brokerId); + } + + private CompletableFuture> getActiveOwnerAsync( + String serviceUnit, + ServiceUnitState state, + Optional owner) { + return deferGetOwnerRequest(serviceUnit) + .thenCompose(newOwner -> { + if (newOwner == null) { + return CompletableFuture.completedFuture(null); + } + + return brokerRegistry.lookupAsync(newOwner) + .thenApply(lookupData -> { + if (lookupData.isPresent()) { + return newOwner; + } else { + throw new IllegalStateException( + "The new owner " + newOwner + " is inactive."); + } + }); + }).whenComplete((__, e) -> { + if (e != null) { + log.error("{} failed to get active owner broker. serviceUnit:{}, state:{}, owner:{}", + brokerId, serviceUnit, state, owner, e); + ownerLookUpCounters.get(state).getFailure().incrementAndGet(); + } + }).thenApply(Optional::ofNullable); } public CompletableFuture> getOwnerAsync(String serviceUnit) { @@ -498,18 +516,22 @@ public CompletableFuture> getOwnerAsync(String serviceUnit) { ownerLookUpCounters.get(state).getTotal().incrementAndGet(); switch (state) { case Owned -> { - return CompletableFuture.completedFuture(Optional.of(data.dstBroker())); + return getActiveOwnerAsync(serviceUnit, state, Optional.of(data.dstBroker())); } case Splitting -> { - return CompletableFuture.completedFuture(Optional.of(data.sourceBroker())); + return getActiveOwnerAsync(serviceUnit, state, Optional.of(data.sourceBroker())); } case Assigning, Releasing -> { - return deferGetOwnerRequest(serviceUnit).whenComplete((__, e) -> { - if (e != null) { - ownerLookUpCounters.get(state).getFailure().incrementAndGet(); - } - }).thenApply( - broker -> broker == null ? Optional.empty() : Optional.of(broker)); + if (isTargetBroker(data.dstBroker())) { + return getActiveOwnerAsync(serviceUnit, state, Optional.of(data.dstBroker())); + } + // If this broker is not the dst broker, return the dst broker as the owner(or empty). + // Clients need to connect(redirect) to the dst broker anyway + // and wait for the dst broker to receive `Owned`. + // This is also required to return getOwnerAsync on the src broker immediately during unloading. + // Otherwise, topic creation(getOwnerAsync) could block unloading bundles, + // if the topic creation(getOwnerAsync) happens during unloading on the src broker. + return CompletableFuture.completedFuture(Optional.ofNullable(data.dstBroker())); } case Init, Free -> { return CompletableFuture.completedFuture(Optional.empty()); @@ -527,6 +549,25 @@ public CompletableFuture> getOwnerAsync(String serviceUnit) { } } + private Optional getOwner(String serviceUnit) { + ServiceUnitStateData data = tableview.get(serviceUnit); + ServiceUnitState state = state(data); + switch (state) { + case Owned -> { + return Optional.of(data.dstBroker()); + } + case Splitting -> { + return Optional.of(data.sourceBroker()); + } + case Init, Free -> { + return Optional.empty(); + } + default -> { + return null; + } + } + } + @Override public Optional getAssigned(String serviceUnit) { if (!validateChannelState(Started, true)) { @@ -656,7 +697,7 @@ private void handle(String serviceUnit, ServiceUnitStateData data) { long totalHandledRequests = getHandlerTotalCounter(data).incrementAndGet(); if (debug()) { log.info("{} received a handle request for serviceUnit:{}, data:{}. totalHandledRequests:{}", - lookupServiceAddress, serviceUnit, data, totalHandledRequests); + brokerId, serviceUnit, data, totalHandledRequests); } ServiceUnitState state = state(data); @@ -715,12 +756,12 @@ private AtomicLong getHandlerCounter(ServiceUnitStateData data, boolean total) { private void log(Throwable e, String serviceUnit, ServiceUnitStateData data, ServiceUnitStateData next) { if (e == null) { - if (log.isDebugEnabled() || isTransferCommand(data)) { + if (debug() || isTransferCommand(data)) { long handlerTotalCount = getHandlerTotalCounter(data).get(); long handlerFailureCount = getHandlerFailureCounter(data).get(); log.info("{} handled {} event for serviceUnit:{}, cur:{}, next:{}, " + "totalHandledRequests:{}, totalFailedRequests:{}", - lookupServiceAddress, getLogEventTag(data), serviceUnit, + brokerId, getLogEventTag(data), serviceUnit, data == null ? "" : data, next == null ? "" : next, handlerTotalCount, handlerFailureCount @@ -731,7 +772,7 @@ lookupServiceAddress, getLogEventTag(data), serviceUnit, long handlerFailureCount = getHandlerFailureCounter(data).incrementAndGet(); log.error("{} failed to handle {} event for serviceUnit:{}, cur:{}, next:{}, " + "totalHandledRequests:{}, totalFailedRequests:{}", - lookupServiceAddress, getLogEventTag(data), serviceUnit, + brokerId, getLogEventTag(data), serviceUnit, data == null ? "" : data, next == null ? "" : next, handlerTotalCount, handlerFailureCount, @@ -754,6 +795,9 @@ private void handleSkippedEvent(String serviceUnit) { private void handleOwnEvent(String serviceUnit, ServiceUnitStateData data) { var getOwnerRequest = getOwnerRequests.remove(serviceUnit); if (getOwnerRequest != null) { + if (debug()) { + log.info("Returned owner request for serviceUnit:{}", serviceUnit); + } getOwnerRequest.complete(data.dstBroker()); } @@ -812,9 +856,14 @@ private void handleFreeEvent(String serviceUnit, ServiceUnitStateData data) { if (getOwnerRequest != null) { getOwnerRequest.complete(null); } - stateChangeListeners.notify(serviceUnit, data, null); + if (isTargetBroker(data.sourceBroker())) { - log(null, serviceUnit, data, null); + stateChangeListeners.notifyOnCompletion( + data.force() ? closeServiceUnit(serviceUnit, true) + : CompletableFuture.completedFuture(0), serviceUnit, data) + .whenComplete((__, e) -> log(e, serviceUnit, data, null)); + } else { + stateChangeListeners.notify(serviceUnit, data, null); } } @@ -864,30 +913,56 @@ private boolean isTargetBroker(String broker) { if (broker == null) { return false; } - return broker.equals(lookupServiceAddress); + return broker.equals(brokerId); } private CompletableFuture deferGetOwnerRequest(String serviceUnit) { + var requested = new MutableObject>(); try { return getOwnerRequests .computeIfAbsent(serviceUnit, k -> { - CompletableFuture future = new CompletableFuture<>(); + var ownerBefore = getOwner(serviceUnit); + if (ownerBefore != null && ownerBefore.isPresent()) { + // Here, we do a quick active check first with the computeIfAbsent lock + brokerRegistry.lookupAsync(ownerBefore.get()).getNow(Optional.empty()) + .ifPresent(__ -> requested.setValue( + CompletableFuture.completedFuture(ownerBefore.get()))); + + if (requested.getValue() != null) { + return requested.getValue(); + } + } + + + CompletableFuture future = + new CompletableFuture().orTimeout(inFlightStateWaitingTimeInMillis, + TimeUnit.MILLISECONDS) + .exceptionally(e -> { + var ownerAfter = getOwner(serviceUnit); + log.warn("{} failed to wait for owner for serviceUnit:{}; Trying to " + + "return the current owner:{}", + brokerId, serviceUnit, ownerAfter, e); + if (ownerAfter == null) { + throw new IllegalStateException(e); + } + return ownerAfter.orElse(null); + }); + if (debug()) { + log.info("{} is waiting for owner for serviceUnit:{}", brokerId, serviceUnit); + } requested.setValue(future); return future; }); } finally { var future = requested.getValue(); if (future != null) { - future.orTimeout(inFlightStateWaitingTimeInMillis + 5 * 1000, TimeUnit.MILLISECONDS) - .whenComplete((v, e) -> { - if (e != null) { - getOwnerRequests.remove(serviceUnit, future); - log.warn("Failed to getOwner for serviceUnit:{}", - serviceUnit, e); - } - } - ); + future.whenComplete((__, e) -> { + getOwnerRequests.remove(serviceUnit); + if (e != null) { + log.warn("{} failed to getOwner for serviceUnit:{}", brokerId, serviceUnit, e); + } + }); } } } @@ -1202,38 +1277,43 @@ private void scheduleCleanup(String broker, long delayInSecs) { private ServiceUnitStateData getOverrideInactiveBrokerStateData(ServiceUnitStateData orphanData, - String selectedBroker, + Optional selectedBroker, String inactiveBroker) { + + + if (selectedBroker.isEmpty()) { + return new ServiceUnitStateData(Free, null, inactiveBroker, + true, getNextVersionId(orphanData)); + } + if (orphanData.state() == Splitting) { - return new ServiceUnitStateData(Splitting, orphanData.dstBroker(), selectedBroker, + return new ServiceUnitStateData(Splitting, orphanData.dstBroker(), selectedBroker.get(), Map.copyOf(orphanData.splitServiceUnitToDestBroker()), true, getNextVersionId(orphanData)); } else { - return new ServiceUnitStateData(Owned, selectedBroker, inactiveBroker, + return new ServiceUnitStateData(Owned, selectedBroker.get(), inactiveBroker, true, getNextVersionId(orphanData)); } } private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanData, String inactiveBroker) { Optional selectedBroker = selectBroker(serviceUnit, inactiveBroker); - if (selectedBroker.isPresent()) { - var override = getOverrideInactiveBrokerStateData( - orphanData, selectedBroker.get(), inactiveBroker); - log.info("Overriding ownership serviceUnit:{} from orphanData:{} to overrideData:{}", - serviceUnit, orphanData, override); - publishOverrideEventAsync(serviceUnit, orphanData, override) - .exceptionally(e -> { - log.error( - "Failed to override the ownership serviceUnit:{} orphanData:{}. " - + "Failed to publish override event. totalCleanupErrorCnt:{}", - serviceUnit, orphanData, totalCleanupErrorCnt.incrementAndGet()); - return null; - }); - } else { - log.error("Failed to override the ownership serviceUnit:{} orphanData:{}. Empty selected broker. " + if (selectedBroker.isEmpty()) { + log.warn("Empty selected broker for ownership serviceUnit:{} orphanData:{}." + "totalCleanupErrorCnt:{}", serviceUnit, orphanData, totalCleanupErrorCnt.incrementAndGet()); } + var override = getOverrideInactiveBrokerStateData(orphanData, selectedBroker, inactiveBroker); + log.info("Overriding ownership serviceUnit:{} from orphanData:{} to overrideData:{}", + serviceUnit, orphanData, override); + publishOverrideEventAsync(serviceUnit, orphanData, override) + .exceptionally(e -> { + log.error( + "Failed to override the ownership serviceUnit:{} orphanData:{}. " + + "Failed to publish override event. totalCleanupErrorCnt:{}", + serviceUnit, orphanData, totalCleanupErrorCnt.incrementAndGet()); + return null; + }); } private void waitForCleanups(String broker, boolean excludeSystemTopics, int maxWaitTimeInMillis) { @@ -1265,7 +1345,7 @@ private void waitForCleanups(String broker, boolean excludeSystemTopics, int max MILLISECONDS.sleep(OWNERSHIP_CLEAN_UP_WAIT_RETRY_DELAY_IN_MILLIS); } catch (InterruptedException e) { log.warn("Interrupted while delaying the next service unit clean-up. Cleaning broker:{}", - lookupServiceAddress); + brokerId); } } } @@ -1335,7 +1415,7 @@ private synchronized void doCleanup(String broker) { broker, cleanupTime, orphanServiceUnitCleanupCnt, - totalCleanupErrorCntStart - totalCleanupErrorCnt.get(), + totalCleanupErrorCnt.get() - totalCleanupErrorCntStart, printCleanupMetrics()); } @@ -1524,7 +1604,7 @@ protected void monitorOwnerships(List brokers) { inactiveBrokers, inactiveBrokers.size(), orphanServiceUnitCleanupCnt, serviceUnitTombstoneCleanupCnt, - totalCleanupErrorCntStart - totalCleanupErrorCnt.get(), + totalCleanupErrorCnt.get() - totalCleanupErrorCntStart, printCleanupMetrics()); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java index 468552db541ec..56238d6528e60 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java @@ -42,14 +42,14 @@ public CompletableFuture> applyIsolationPoliciesAsync(Map, private final BrokerHostUsage brokerHostUsage; - private final String lookupServiceAddress; + private final String brokerId; @Getter private final BrokerLoadData localData; @@ -67,10 +67,10 @@ public class BrokerLoadDataReporter implements LoadDataReporter, private long tombstoneDelayInMillis; public BrokerLoadDataReporter(PulsarService pulsar, - String lookupServiceAddress, + String brokerId, LoadDataStore brokerLoadDataStore) { this.brokerLoadDataStore = brokerLoadDataStore; - this.lookupServiceAddress = lookupServiceAddress; + this.brokerId = brokerId; this.pulsar = pulsar; this.conf = this.pulsar.getConfiguration(); if (SystemUtils.IS_OS_LINUX) { @@ -111,7 +111,7 @@ public CompletableFuture reportAsync(boolean force) { log.info("publishing load report:{}", localData.toString(conf)); } CompletableFuture future = - this.brokerLoadDataStore.pushAsync(this.lookupServiceAddress, newLoadData); + this.brokerLoadDataStore.pushAsync(this.brokerId, newLoadData); future.whenComplete((__, ex) -> { if (ex == null) { localData.setReportedAt(System.currentTimeMillis()); @@ -185,7 +185,7 @@ protected void tombstone() { } var lastSuccessfulTombstonedAt = lastTombstonedAt; lastTombstonedAt = now; // dedup first - brokerLoadDataStore.removeAsync(lookupServiceAddress) + brokerLoadDataStore.removeAsync(brokerId) .whenComplete((__, e) -> { if (e != null) { log.error("Failed to clean broker load data.", e); @@ -209,13 +209,13 @@ public void handleEvent(String serviceUnit, ServiceUnitStateData data, Throwable ServiceUnitState state = ServiceUnitStateData.state(data); switch (state) { case Releasing, Splitting -> { - if (StringUtils.equals(data.sourceBroker(), lookupServiceAddress)) { + if (StringUtils.equals(data.sourceBroker(), brokerId)) { localData.clear(); tombstone(); } } case Owned -> { - if (StringUtils.equals(data.dstBroker(), lookupServiceAddress)) { + if (StringUtils.equals(data.dstBroker(), brokerId)) { localData.clear(); tombstone(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java index 0fa37d3687c20..43e05ad1ac972 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java @@ -41,7 +41,7 @@ public class TopBundleLoadDataReporter implements LoadDataReporter bundleLoadDataStore; @@ -53,10 +53,10 @@ public class TopBundleLoadDataReporter implements LoadDataReporter bundleLoadDataStore) { this.pulsar = pulsar; - this.lookupServiceAddress = lookupServiceAddress; + this.brokerId = brokerId; this.bundleLoadDataStore = bundleLoadDataStore; this.lastBundleStatsUpdatedAt = 0; this.topKBundles = new TopKBundles(pulsar); @@ -88,7 +88,7 @@ public CompletableFuture reportAsync(boolean force) { if (ExtensibleLoadManagerImpl.debug(pulsar.getConfiguration(), log)) { log.info("Reporting TopBundlesLoadData:{}", topKBundles.getLoadData()); } - return this.bundleLoadDataStore.pushAsync(lookupServiceAddress, topKBundles.getLoadData()) + return this.bundleLoadDataStore.pushAsync(brokerId, topKBundles.getLoadData()) .exceptionally(e -> { log.error("Failed to report top-bundles load data.", e); return null; @@ -106,7 +106,7 @@ protected void tombstone() { } var lastSuccessfulTombstonedAt = lastTombstonedAt; lastTombstonedAt = now; // dedup first - bundleLoadDataStore.removeAsync(lookupServiceAddress) + bundleLoadDataStore.removeAsync(brokerId) .whenComplete((__, e) -> { if (e != null) { log.error("Failed to clean broker load data.", e); @@ -129,12 +129,12 @@ public void handleEvent(String serviceUnit, ServiceUnitStateData data, Throwable ServiceUnitState state = ServiceUnitStateData.state(data); switch (state) { case Releasing, Splitting -> { - if (StringUtils.equals(data.sourceBroker(), lookupServiceAddress)) { + if (StringUtils.equals(data.sourceBroker(), brokerId)) { tombstone(); } } case Owned -> { - if (StringUtils.equals(data.dstBroker(), lookupServiceAddress)) { + if (StringUtils.equals(data.dstBroker(), brokerId)) { tombstone(); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java index d6c754c90fcf6..218f57932a56b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java @@ -219,9 +219,8 @@ private static NamespaceUnloadStrategy createNamespaceUnloadStrategy(PulsarServi Thread.currentThread().getContextClassLoader()); log.info("Created namespace unload strategy:{}", unloadStrategy.getClass().getCanonicalName()); } catch (Exception e) { - log.error("Error when trying to create namespace unload strategy: {}", - conf.getLoadBalancerLoadPlacementStrategy(), e); - log.error("create namespace unload strategy failed. using TransferShedder instead."); + log.error("Error when trying to create namespace unload strategy: {}. Using {} instead.", + conf.getLoadBalancerLoadSheddingStrategy(), TransferShedder.class.getCanonicalName(), e); unloadStrategy = new TransferShedder(); } unloadStrategy.initialize(pulsar); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreFactory.java index 18f39abd76b76..bcb2657c67f05 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreFactory.java @@ -18,15 +18,16 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.store; -import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.broker.PulsarService; /** * The load data store factory, use to create the load data store. */ public class LoadDataStoreFactory { - public static LoadDataStore create(PulsarClient client, String name, Class clazz) + public static LoadDataStore create(PulsarService pulsar, String name, + Class clazz) throws LoadDataStoreException { - return new TableViewLoadDataStoreImpl<>(client, name, clazz); + return new TableViewLoadDataStoreImpl<>(pulsar, name, clazz); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java index 56afbef04565c..d916e91716223 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java @@ -23,34 +23,46 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.TableView; -import org.apache.pulsar.common.util.FutureUtil; /** * The load data store, base on {@link TableView }. * * @param Load data type. */ +@Slf4j public class TableViewLoadDataStoreImpl implements LoadDataStore { + private static final long LOAD_DATA_REPORT_UPDATE_MAX_INTERVAL_MULTIPLIER_BEFORE_RESTART = 2; + private volatile TableView tableView; + private volatile long tableViewLastUpdateTimestamp; private volatile Producer producer; + private final ServiceConfiguration conf; + private final PulsarClient client; private final String topic; private final Class clazz; - public TableViewLoadDataStoreImpl(PulsarClient client, String topic, Class clazz) throws LoadDataStoreException { + public TableViewLoadDataStoreImpl(PulsarService pulsar, String topic, Class clazz) + throws LoadDataStoreException { try { - this.client = client; + this.conf = pulsar.getConfiguration(); + this.client = pulsar.getClient(); this.topic = topic; this.clazz = clazz; } catch (Exception e) { @@ -60,40 +72,36 @@ public TableViewLoadDataStoreImpl(PulsarClient client, String topic, Class cl @Override public synchronized CompletableFuture pushAsync(String key, T loadData) { - if (producer == null) { - return FutureUtil.failedFuture(new IllegalStateException("producer has not been started")); - } + validateProducer(); return producer.newMessage().key(key).value(loadData).sendAsync().thenAccept(__ -> {}); } @Override public synchronized CompletableFuture removeAsync(String key) { - if (producer == null) { - return FutureUtil.failedFuture(new IllegalStateException("producer has not been started")); - } + validateProducer(); return producer.newMessage().key(key).value(null).sendAsync().thenAccept(__ -> {}); } @Override public synchronized Optional get(String key) { - validateTableViewStart(); + validateTableView(); return Optional.ofNullable(tableView.get(key)); } @Override public synchronized void forEach(BiConsumer action) { - validateTableViewStart(); + validateTableView(); tableView.forEach(action); } public synchronized Set> entrySet() { - validateTableViewStart(); + validateTableView(); return tableView.entrySet(); } @Override public synchronized int size() { - validateTableViewStart(); + validateTableView(); return tableView.size(); } @@ -116,6 +124,8 @@ public synchronized void startTableView() throws LoadDataStoreException { if (tableView == null) { try { tableView = client.newTableViewBuilder(Schema.JSON(clazz)).topic(topic).create(); + tableView.forEachAndListen((k, v) -> + tableViewLastUpdateTimestamp = System.currentTimeMillis()); } catch (PulsarClientException e) { tableView = null; throw new LoadDataStoreException(e); @@ -150,9 +160,48 @@ public synchronized void init() throws IOException { start(); } - private synchronized void validateTableViewStart() { + private void validateProducer() { + if (producer == null || !producer.isConnected()) { + try { + if (producer != null) { + producer.close(); + } + producer = null; + startProducer(); + log.info("Restarted producer on {}", topic); + } catch (Exception e) { + log.error("Failed to restart producer on {}", topic, e); + throw new RuntimeException(e); + } + } + } + + private void validateTableView() { + String restartReason = null; + if (tableView == null) { - throw new IllegalStateException("table view has not been started"); + restartReason = "table view is null"; + } else { + long inactiveDuration = System.currentTimeMillis() - tableViewLastUpdateTimestamp; + long threshold = TimeUnit.MINUTES.toMillis(conf.getLoadBalancerReportUpdateMaxIntervalMinutes()) + * LOAD_DATA_REPORT_UPDATE_MAX_INTERVAL_MULTIPLIER_BEFORE_RESTART; + if (inactiveDuration > threshold) { + restartReason = String.format("inactiveDuration=%d secs > threshold = %d secs", + TimeUnit.MILLISECONDS.toSeconds(inactiveDuration), + TimeUnit.MILLISECONDS.toSeconds(threshold)); + } + } + + if (StringUtils.isNotBlank(restartReason)) { + tableViewLastUpdateTimestamp = 0; + try { + closeTableView(); + startTableView(); + log.info("Restarted tableview on {}, {}", topic, restartReason); + } catch (Exception e) { + log.error("Failed to restart tableview on {}", topic, e); + throw new RuntimeException(e); + } } } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java index 5f2e4b1f25d8b..3d627db6cfa9e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java @@ -21,8 +21,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.apache.pulsar.common.stats.JvmMetrics.getJvmDirectMemoryUsed; import io.netty.util.concurrent.FastThreadLocal; -import java.net.MalformedURLException; -import java.net.URL; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -106,59 +104,56 @@ public static void applyNamespacePolicies(final ServiceUnitId serviceUnit, if (isIsolationPoliciesPresent) { LOG.debug("Isolation Policies Present for namespace - [{}]", namespace.toString()); } - for (final String broker : availableBrokers) { - final String brokerUrlString = String.format("http://%s", broker); - URL brokerUrl; + for (final String brokerId : availableBrokers) { + String brokerHost; try { - brokerUrl = new URL(brokerUrlString); - } catch (MalformedURLException e) { - LOG.error("Unable to parse brokerUrl from ResourceUnitId", e); + brokerHost = parseBrokerHost(brokerId); + } catch (IllegalArgumentException e) { + LOG.error("Unable to parse host from {}", brokerId, e); continue; } // todo: in future check if the resource unit has resources to take the namespace if (isIsolationPoliciesPresent) { // note: serviceUnitID is namespace name and ResourceID is brokerName - if (policies.isPrimaryBroker(namespace, brokerUrl.getHost())) { - primariesCache.add(broker); + if (policies.isPrimaryBroker(namespace, brokerHost)) { + primariesCache.add(brokerId); if (LOG.isDebugEnabled()) { LOG.debug("Added Primary Broker - [{}] as possible Candidates for" - + " namespace - [{}] with policies", brokerUrl.getHost(), namespace.toString()); + + " namespace - [{}] with policies", brokerHost, namespace.toString()); } - } else if (policies.isSecondaryBroker(namespace, brokerUrl.getHost())) { - secondaryCache.add(broker); + } else if (policies.isSecondaryBroker(namespace, brokerHost)) { + secondaryCache.add(brokerId); if (LOG.isDebugEnabled()) { LOG.debug( "Added Shared Broker - [{}] as possible " + "Candidates for namespace - [{}] with policies", - brokerUrl.getHost(), namespace.toString()); + brokerHost, namespace.toString()); } } else { if (LOG.isDebugEnabled()) { LOG.debug("Skipping Broker - [{}] not primary broker and not shared" + " for namespace - [{}] ", - brokerUrl.getHost(), namespace.toString()); + brokerHost, namespace.toString()); } } } else { // non-persistent topic can be assigned to only those brokers that enabled for non-persistent topic - if (isNonPersistentTopic - && !brokerTopicLoadingPredicate.isEnableNonPersistentTopics(brokerUrlString)) { + if (isNonPersistentTopic && !brokerTopicLoadingPredicate.isEnableNonPersistentTopics(brokerId)) { if (LOG.isDebugEnabled()) { LOG.debug("Filter broker- [{}] because it doesn't support non-persistent namespace - [{}]", - brokerUrl.getHost(), namespace.toString()); + brokerHost, namespace.toString()); } - } else if (!isNonPersistentTopic - && !brokerTopicLoadingPredicate.isEnablePersistentTopics(brokerUrlString)) { + } else if (!isNonPersistentTopic && !brokerTopicLoadingPredicate.isEnablePersistentTopics(brokerId)) { // persistent topic can be assigned to only brokers that enabled for persistent-topic if (LOG.isDebugEnabled()) { LOG.debug("Filter broker- [{}] because broker only supports non-persistent namespace - [{}]", - brokerUrl.getHost(), namespace.toString()); + brokerHost, namespace.toString()); } - } else if (policies.isSharedBroker(brokerUrl.getHost())) { - secondaryCache.add(broker); + } else if (policies.isSharedBroker(brokerHost)) { + secondaryCache.add(brokerId); if (LOG.isDebugEnabled()) { LOG.debug("Added Shared Broker - [{}] as possible Candidates for namespace - [{}]", - brokerUrl.getHost(), namespace.toString()); + brokerHost, namespace.toString()); } } } @@ -181,6 +176,16 @@ public static void applyNamespacePolicies(final ServiceUnitId serviceUnit, } } + private static String parseBrokerHost(String brokerId) { + // use last index to support ipv6 addresses + int lastIdx = brokerId.lastIndexOf(':'); + if (lastIdx > -1) { + return brokerId.substring(0, lastIdx); + } else { + throw new IllegalArgumentException("Invalid brokerId: " + brokerId); + } + } + public static CompletableFuture> applyNamespacePoliciesAsync( final ServiceUnitId serviceUnit, final SimpleResourceAllocationPolicies policies, final Set availableBrokers, final BrokerTopicLoadingPredicate brokerTopicLoadingPredicate) { @@ -199,59 +204,57 @@ public static CompletableFuture> applyNamespacePoliciesAsync( LOG.debug("Isolation Policies Present for namespace - [{}]", namespace.toString()); } } - for (final String broker : availableBrokers) { - final String brokerUrlString = String.format("http://%s", broker); - URL brokerUrl; + for (final String brokerId : availableBrokers) { + String brokerHost; try { - brokerUrl = new URL(brokerUrlString); - } catch (MalformedURLException e) { - LOG.error("Unable to parse brokerUrl from ResourceUnitId", e); + brokerHost = parseBrokerHost(brokerId); + } catch (IllegalArgumentException e) { + LOG.error("Unable to parse host from {}", brokerId, e); continue; } // todo: in future check if the resource unit has resources to take the namespace if (isIsolationPoliciesPresent) { // note: serviceUnitID is namespace name and ResourceID is brokerName - if (policies.isPrimaryBroker(namespace, brokerUrl.getHost())) { - primariesCache.add(broker); + if (policies.isPrimaryBroker(namespace, brokerHost)) { + primariesCache.add(brokerId); if (LOG.isDebugEnabled()) { LOG.debug("Added Primary Broker - [{}] as possible Candidates for" - + " namespace - [{}] with policies", brokerUrl.getHost(), namespace.toString()); + + " namespace - [{}] with policies", brokerHost, namespace.toString()); } - } else if (policies.isSecondaryBroker(namespace, brokerUrl.getHost())) { - secondaryCache.add(broker); + } else if (policies.isSecondaryBroker(namespace, brokerHost)) { + secondaryCache.add(brokerId); if (LOG.isDebugEnabled()) { LOG.debug( "Added Shared Broker - [{}] as possible " + "Candidates for namespace - [{}] with policies", - brokerUrl.getHost(), namespace.toString()); + brokerHost, namespace.toString()); } } else { if (LOG.isDebugEnabled()) { LOG.debug("Skipping Broker - [{}] not primary broker and not shared" - + " for namespace - [{}] ", brokerUrl.getHost(), namespace.toString()); + + " for namespace - [{}] ", brokerHost, namespace.toString()); } } } else { // non-persistent topic can be assigned to only those brokers that enabled for non-persistent topic - if (isNonPersistentTopic - && !brokerTopicLoadingPredicate.isEnableNonPersistentTopics(brokerUrlString)) { + if (isNonPersistentTopic && !brokerTopicLoadingPredicate.isEnableNonPersistentTopics(brokerId)) { if (LOG.isDebugEnabled()) { LOG.debug("Filter broker- [{}] because it doesn't support non-persistent namespace - [{}]", - brokerUrl.getHost(), namespace.toString()); + brokerId, namespace.toString()); } - } else if (!isNonPersistentTopic - && !brokerTopicLoadingPredicate.isEnablePersistentTopics(brokerUrlString)) { + } else if (!isNonPersistentTopic && !brokerTopicLoadingPredicate + .isEnablePersistentTopics(brokerId)) { // persistent topic can be assigned to only brokers that enabled for persistent-topic if (LOG.isDebugEnabled()) { LOG.debug("Filter broker- [{}] because broker only supports non-persistent " - + "namespace - [{}]", brokerUrl.getHost(), namespace.toString()); + + "namespace - [{}]", brokerId, namespace.toString()); } - } else if (policies.isSharedBroker(brokerUrl.getHost())) { - secondaryCache.add(broker); + } else if (policies.isSharedBroker(brokerHost)) { + secondaryCache.add(brokerId); if (LOG.isDebugEnabled()) { LOG.debug("Added Shared Broker - [{}] as possible Candidates for namespace - [{}]", - brokerUrl.getHost(), namespace.toString()); + brokerHost, namespace.toString()); } } } @@ -762,9 +765,9 @@ public static boolean shouldAntiAffinityNamespaceUnload( } public interface BrokerTopicLoadingPredicate { - boolean isEnablePersistentTopics(String brokerUrl); + boolean isEnablePersistentTopics(String brokerId); - boolean isEnableNonPersistentTopics(String brokerUrl); + boolean isEnableNonPersistentTopics(String brokerId); } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index 4ecdfefbdd041..974d75d60b203 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -208,15 +208,15 @@ public ModularLoadManagerImpl() { this.bundleBrokerAffinityMap = new ConcurrentHashMap<>(); this.brokerTopicLoadingPredicate = new BrokerTopicLoadingPredicate() { @Override - public boolean isEnablePersistentTopics(String brokerUrl) { - final BrokerData brokerData = loadData.getBrokerData().get(brokerUrl.replace("http://", "")); + public boolean isEnablePersistentTopics(String brokerId) { + final BrokerData brokerData = loadData.getBrokerData().get(brokerId); return brokerData != null && brokerData.getLocalData() != null && brokerData.getLocalData().isPersistentTopicsEnabled(); } @Override - public boolean isEnableNonPersistentTopics(String brokerUrl) { - final BrokerData brokerData = loadData.getBrokerData().get(brokerUrl.replace("http://", "")); + public boolean isEnableNonPersistentTopics(String brokerId) { + final BrokerData brokerData = loadData.getBrokerData().get(brokerId); return brokerData != null && brokerData.getLocalData() != null && brokerData.getLocalData().isNonPersistentTopicsEnabled(); } @@ -977,14 +977,14 @@ public void start() throws PulsarServerException { localData.setNonPersistentTopicsEnabled(pulsar.getConfiguration().isEnableNonPersistentTopics()); localData.setLoadManagerClassName(conf.getLoadManagerClassName()); - String lookupServiceAddress = pulsar.getLookupServiceAddress(); - brokerZnodePath = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + lookupServiceAddress; + String brokerId = pulsar.getBrokerId(); + brokerZnodePath = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + brokerId; updateLocalBrokerData(); brokerDataLock = brokersData.acquireLock(brokerZnodePath, localData).join(); pulsarResources.getLoadBalanceResources() .getBrokerTimeAverageDataResources() - .updateTimeAverageBrokerData(lookupServiceAddress, new TimeAverageBrokerData()) + .updateTimeAverageBrokerData(brokerId, new TimeAverageBrokerData()) .join(); updateAll(); } catch (Exception e) { @@ -1212,7 +1212,6 @@ public String setNamespaceBundleAffinity(String bundle, String broker) { if (StringUtils.isBlank(broker)) { return this.bundleBrokerAffinityMap.remove(bundle); } - broker = broker.replaceFirst("http[s]?://", ""); return this.bundleBrokerAffinityMap.put(bundle, broker); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java index c61d39cf3159a..c8d81bda1bc13 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java @@ -19,7 +19,6 @@ package org.apache.pulsar.broker.loadbalance.impl; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -32,7 +31,6 @@ import org.apache.pulsar.common.naming.ServiceUnitId; import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.policies.data.loadbalancer.LoadManagerReport; -import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; /** * Wrapper class allowing classes of instance ModularLoadManager to be compatible with the interface LoadManager. @@ -75,20 +73,6 @@ public Optional getLeastLoaded(final ServiceUnitId serviceUnit) { return leastLoadedBroker.map(this::buildBrokerResourceUnit); } - private String getBrokerWebServiceUrl(String broker) { - LocalBrokerData localData = (loadManager).getBrokerLocalData(broker); - if (localData != null) { - return localData.getWebServiceUrl() != null ? localData.getWebServiceUrl() - : localData.getWebServiceUrlTls(); - } - return String.format("http://%s", broker); - } - - private String getBrokerZnodeName(String broker, String webServiceUrl) { - String scheme = webServiceUrl.substring(0, webServiceUrl.indexOf("://")); - return String.format("%s://%s", scheme, broker); - } - @Override public List getLoadBalancingMetrics() { return loadManager.getLoadBalancingMetrics(); @@ -149,10 +133,7 @@ public CompletableFuture> getAvailableBrokersAsync() { } private SimpleResourceUnit buildBrokerResourceUnit (String broker) { - String webServiceUrl = getBrokerWebServiceUrl(broker); - String brokerZnodeName = getBrokerZnodeName(broker, webServiceUrl); - return new SimpleResourceUnit(webServiceUrl, - new PulsarResourceDescription(), Map.of(ResourceUnit.PROPERTY_KEY_BROKER_ZNODE_NAME, brokerZnodeName)); + return new SimpleResourceUnit(broker, new PulsarResourceDescription()); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java index ee60595a485c4..be0580808cafb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java @@ -211,15 +211,15 @@ public SimpleLoadManagerImpl() { .build(); this.brokerTopicLoadingPredicate = new BrokerTopicLoadingPredicate() { @Override - public boolean isEnablePersistentTopics(String brokerUrl) { - ResourceUnit ru = new SimpleResourceUnit(brokerUrl, new PulsarResourceDescription()); + public boolean isEnablePersistentTopics(String brokerId) { + ResourceUnit ru = new SimpleResourceUnit(brokerId, new PulsarResourceDescription()); LoadReport loadReport = currentLoadReports.get(ru); return loadReport != null && loadReport.isPersistentTopicsEnabled(); } @Override - public boolean isEnableNonPersistentTopics(String brokerUrl) { - ResourceUnit ru = new SimpleResourceUnit(brokerUrl, new PulsarResourceDescription()); + public boolean isEnableNonPersistentTopics(String brokerId) { + ResourceUnit ru = new SimpleResourceUnit(brokerId, new PulsarResourceDescription()); LoadReport loadReport = currentLoadReports.get(ru); return loadReport != null && loadReport.isNonPersistentTopicsEnabled(); } @@ -266,8 +266,8 @@ public SimpleLoadManagerImpl(PulsarService pulsar) { @Override public void start() throws PulsarServerException { // Register the brokers in metadata store - String lookupServiceAddress = pulsar.getLookupServiceAddress(); - String brokerLockPath = LOADBALANCE_BROKERS_ROOT + "/" + lookupServiceAddress; + String brokerId = pulsar.getBrokerId(); + String brokerLockPath = LOADBALANCE_BROKERS_ROOT + "/" + brokerId; try { LoadReport loadReport = null; @@ -653,7 +653,6 @@ public void writeResourceQuotasToZooKeeper() throws Exception { */ private synchronized void doLoadRanking() { ResourceUnitRanking.setCpuUsageByMsgRate(this.realtimeCpuLoadFactor); - String hostname = pulsar.getAdvertisedAddress(); String strategy = this.getLoadBalancerPlacementStrategy(); log.info("doLoadRanking - load balancing strategy: {}", strategy); if (!currentLoadReports.isEmpty()) { @@ -702,8 +701,8 @@ private synchronized void doLoadRanking() { } // update metrics - if (resourceUnit.getResourceId().contains(hostname)) { - updateLoadBalancingMetrics(hostname, finalRank, ranking); + if (resourceUnit.getResourceId().equals(pulsar.getBrokerId())) { + updateLoadBalancingMetrics(pulsar.getAdvertisedAddress(), finalRank, ranking); } } updateBrokerToNamespaceToBundle(); @@ -711,7 +710,7 @@ private synchronized void doLoadRanking() { this.resourceUnitRankings = newResourceUnitRankings; } else { log.info("Leader broker[{}] No ResourceUnits to rank this run, Using Old Ranking", - pulsar.getSafeWebServiceAddress()); + pulsar.getBrokerId()); } } @@ -855,7 +854,7 @@ private synchronized ResourceUnit findBrokerForPlacement(Multimap ConcurrentOpenHashMap.>newBuilder() .build()) @@ -876,7 +875,7 @@ private Multimap getFinalCandidates(ServiceUnitId serviceUni availableBrokersCache.clear(); for (final Set resourceUnits : availableBrokers.values()) { for (final ResourceUnit resourceUnit : resourceUnits) { - availableBrokersCache.add(resourceUnit.getResourceId().replace("http://", "")); + availableBrokersCache.add(resourceUnit.getResourceId()); } } brokerCandidateCache.clear(); @@ -899,7 +898,7 @@ private Multimap getFinalCandidates(ServiceUnitId serviceUni final Long rank = entry.getKey(); final Set resourceUnits = entry.getValue(); for (final ResourceUnit resourceUnit : resourceUnits) { - if (brokerCandidateCache.contains(resourceUnit.getResourceId().replace("http://", ""))) { + if (brokerCandidateCache.contains(resourceUnit.getResourceId())) { result.put(rank, resourceUnit); } } @@ -928,8 +927,7 @@ private Map> getAvailableBrokers(ServiceUnitId serviceUn availableBrokers = new HashMap<>(); for (String broker : activeBrokers) { - ResourceUnit resourceUnit = new SimpleResourceUnit(String.format("http://%s", broker), - new PulsarResourceDescription()); + ResourceUnit resourceUnit = new SimpleResourceUnit(broker, new PulsarResourceDescription()); availableBrokers.computeIfAbsent(0L, key -> new TreeSet<>()).add(resourceUnit); } log.info("Choosing at random from broker list: [{}]", availableBrokers.values()); @@ -956,7 +954,7 @@ private synchronized ResourceUnit getLeastLoadedBroker(ServiceUnitId serviceUnit Iterator> candidateIterator = finalCandidates.entries().iterator(); while (candidateIterator.hasNext()) { Map.Entry candidate = candidateIterator.next(); - String candidateBrokerName = candidate.getValue().getResourceId().replace("http://", ""); + String candidateBrokerName = candidate.getValue().getResourceId(); if (!activeBrokers.contains(candidateBrokerName)) { candidateIterator.remove(); // Current candidate points to an inactive broker, so remove it } @@ -1005,8 +1003,7 @@ private void updateRanking() { try { String key = String.format("%s/%s", LOADBALANCE_BROKERS_ROOT, broker); LoadReport lr = loadReports.readLock(key).join().get(); - ResourceUnit ru = new SimpleResourceUnit(String.format("http://%s", lr.getName()), - fromLoadReport(lr)); + ResourceUnit ru = new SimpleResourceUnit(lr.getName(), fromLoadReport(lr)); this.currentLoadReports.put(ru, lr); } catch (Exception e) { log.warn("Error reading load report from Cache for broker - [{}], [{}]", broker, e); @@ -1078,7 +1075,7 @@ private LoadReport generateLoadReportForcefully() throws Exception { loadReport.setProtocols(pulsar.getProtocolDataToAdvertise()); loadReport.setNonPersistentTopicsEnabled(pulsar.getConfiguration().isEnableNonPersistentTopics()); loadReport.setPersistentTopicsEnabled(pulsar.getConfiguration().isEnablePersistentTopics()); - loadReport.setName(pulsar.getLookupServiceAddress()); + loadReport.setName(pulsar.getBrokerId()); loadReport.setBrokerVersionString(pulsar.getBrokerVersion()); SystemResourceUsage systemResourceUsage = this.getSystemResourceUsage(); @@ -1121,8 +1118,8 @@ private LoadReport generateLoadReportForcefully() throws Exception { loadReport.setAllocatedMsgRateIn(allocatedQuota.getMsgRateIn()); loadReport.setAllocatedMsgRateOut(allocatedQuota.getMsgRateOut()); - final ResourceUnit resourceUnit = new SimpleResourceUnit( - String.format("http://%s", loadReport.getName()), fromLoadReport(loadReport)); + final ResourceUnit resourceUnit = + new SimpleResourceUnit(loadReport.getName(), fromLoadReport(loadReport)); Set preAllocatedBundles; if (resourceUnitRankings.containsKey(resourceUnit)) { preAllocatedBundles = resourceUnitRankings.get(resourceUnit).getPreAllocatedBundles(); @@ -1277,7 +1274,7 @@ private synchronized void updateBrokerToNamespaceToBundle() { final Set preallocatedBundles = resourceUnitRankings.get(resourceUnit).getPreAllocatedBundles(); final ConcurrentOpenHashMap> namespaceToBundleRange = brokerToNamespaceToBundleRange - .computeIfAbsent(broker.replace("http://", ""), + .computeIfAbsent(broker, k -> ConcurrentOpenHashMap.>newBuilder() .build()); @@ -1455,7 +1452,6 @@ public String setNamespaceBundleAffinity(String bundle, String broker) { if (StringUtils.isBlank(broker)) { return this.bundleBrokerAffinityMap.remove(bundle); } - broker = broker.replaceFirst("http[s]?://", ""); return this.bundleBrokerAffinityMap.put(bundle, broker); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java index c4a39cd0d4455..7b2c777414884 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java @@ -30,6 +30,7 @@ import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Response; import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; @@ -333,13 +334,16 @@ public static CompletableFuture lookupTopicAsync(PulsarService pulsarSe private static void handleLookupError(CompletableFuture lookupFuture, String topicName, String clientAppId, long requestId, Throwable ex){ - final Throwable unwrapEx = FutureUtil.unwrapCompletionException(ex); + Throwable unwrapEx = FutureUtil.unwrapCompletionException(ex); final String errorMsg = unwrapEx.getMessage(); + if (unwrapEx instanceof PulsarServerException) { + unwrapEx = FutureUtil.unwrapCompletionException(unwrapEx.getCause()); + } if (unwrapEx instanceof IllegalStateException) { // Current broker still hold the bundle's lock, but the bundle is being unloading. log.info("Failed to lookup {} for topic {} with error {}", clientAppId, topicName, errorMsg); lookupFuture.complete(newLookupErrorResponse(ServerError.MetadataError, errorMsg, requestId)); - } else if (unwrapEx instanceof MetadataStoreException){ + } else if (unwrapEx instanceof MetadataStoreException) { // Load bundle ownership or acquire lock failed. // Differ with "IllegalStateException", print warning log. log.warn("Failed to lookup {} for topic {} with error {}", clientAppId, topicName, errorMsg); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 4a54d4e090852..b55eda150afe6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -192,7 +192,7 @@ public CompletableFuture> getBrokerServiceUrlAsync(TopicN return findRedirectLookupResultAsync(bundle).thenCompose(optResult -> { if (optResult.isPresent()) { LOG.info("[{}] Redirect lookup request to {} for topic {}", - pulsar.getSafeWebServiceAddress(), optResult.get(), topic); + pulsar.getBrokerId(), optResult.get(), topic); return CompletableFuture.completedFuture(optResult); } if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { @@ -298,7 +298,7 @@ private CompletableFuture> internalGetWebServiceUrl(@Nullable Serv return findRedirectLookupResultAsync(bundle).thenCompose(optResult -> { if (optResult.isPresent()) { LOG.info("[{}] Redirect lookup request to {} for topic {}", - pulsar.getSafeWebServiceAddress(), optResult.get(), topic); + pulsar.getBrokerId(), optResult.get(), topic); try { LookupData lookupData = optResult.get().getLookupData(); final String redirectUrl = options.isRequestHttps() @@ -338,17 +338,17 @@ private CompletableFuture> internalGetWebServiceUrl(@Nullable Serv * @throws PulsarServerException if an unexpected error occurs */ public void registerBootstrapNamespaces() throws PulsarServerException { - String lookupServiceAddress = pulsar.getLookupServiceAddress(); + String brokerId = pulsar.getBrokerId(); // ensure that we own the heartbeat namespace - if (registerNamespace(getHeartbeatNamespace(lookupServiceAddress, config), true)) { + if (registerNamespace(getHeartbeatNamespace(brokerId, config), true)) { LOG.info("added heartbeat namespace name in local cache: ns={}", - getHeartbeatNamespace(lookupServiceAddress, config)); + getHeartbeatNamespace(brokerId, config)); } // ensure that we own the heartbeat namespace - if (registerNamespace(getHeartbeatNamespaceV2(lookupServiceAddress, config), true)) { + if (registerNamespace(getHeartbeatNamespaceV2(brokerId, config), true)) { LOG.info("added heartbeat namespace name in local cache: ns={}", - getHeartbeatNamespaceV2(lookupServiceAddress, config)); + getHeartbeatNamespaceV2(brokerId, config)); } // we may not need strict ownership checking for bootstrap names for now @@ -506,7 +506,6 @@ private void searchForCandidateBroker(NamespaceBundle bundle, return; } String candidateBroker; - String candidateBrokerAdvertisedAddr = null; LeaderElectionService les = pulsar.getLeaderElectionService(); if (les == null) { @@ -541,14 +540,14 @@ private void searchForCandidateBroker(NamespaceBundle bundle, if (options.isAuthoritative()) { // leader broker already assigned the current broker as owner - candidateBroker = pulsar.getSafeWebServiceAddress(); + candidateBroker = pulsar.getBrokerId(); } else { LoadManager loadManager = this.loadManager.get(); boolean makeLoadManagerDecisionOnThisBroker = !loadManager.isCentralized() || les.isLeader(); if (!makeLoadManagerDecisionOnThisBroker) { // If leader is not active, fallback to pick the least loaded from current broker loadmanager boolean leaderBrokerActive = currentLeader.isPresent() - && isBrokerActive(currentLeader.get().getServiceUrl()); + && isBrokerActive(currentLeader.get().getBrokerId()); if (!leaderBrokerActive) { makeLoadManagerDecisionOnThisBroker = true; if (currentLeader.isEmpty()) { @@ -567,7 +566,7 @@ private void searchForCandidateBroker(NamespaceBundle bundle, } } if (makeLoadManagerDecisionOnThisBroker) { - Optional> availableBroker = getLeastLoadedFromLoadManager(bundle); + Optional availableBroker = getLeastLoadedFromLoadManager(bundle); if (availableBroker.isEmpty()) { LOG.warn("Load manager didn't return any available broker. " + "Returning empty result to lookup. NamespaceBundle[{}]", @@ -575,12 +574,11 @@ private void searchForCandidateBroker(NamespaceBundle bundle, lookupFuture.complete(Optional.empty()); return; } - candidateBroker = availableBroker.get().getLeft(); - candidateBrokerAdvertisedAddr = availableBroker.get().getRight(); + candidateBroker = availableBroker.get(); authoritativeRedirect = true; } else { // forward to leader broker to make assignment - candidateBroker = currentLeader.get().getServiceUrl(); + candidateBroker = currentLeader.get().getBrokerId(); } } } @@ -593,7 +591,7 @@ private void searchForCandidateBroker(NamespaceBundle bundle, try { Objects.requireNonNull(candidateBroker); - if (candidateBroker.equals(pulsar.getSafeWebServiceAddress())) { + if (candidateBroker.equals(pulsar.getBrokerId())) { // Load manager decided that the local broker should try to become the owner ownershipCache.tryAcquiringOwnership(bundle).thenAccept(ownerInfo -> { if (ownerInfo.isDisabled()) { @@ -644,8 +642,7 @@ private void searchForCandidateBroker(NamespaceBundle bundle, } // Now setting the redirect url - createLookupResult(candidateBrokerAdvertisedAddr == null ? candidateBroker - : candidateBrokerAdvertisedAddr, authoritativeRedirect, options.getAdvertisedListenerName()) + createLookupResult(candidateBroker, authoritativeRedirect, options.getAdvertisedListenerName()) .thenAccept(lookupResult -> lookupFuture.complete(Optional.of(lookupResult))) .exceptionally(ex -> { lookupFuture.completeExceptionally(ex); @@ -665,7 +662,7 @@ public CompletableFuture createLookupResult(String candidateBroker CompletableFuture lookupFuture = new CompletableFuture<>(); try { checkArgument(StringUtils.isNotBlank(candidateBroker), "Lookup broker can't be null %s", candidateBroker); - String path = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + parseHostAndPort(candidateBroker); + String path = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + candidateBroker; localBrokerDataCache.get(path).thenAccept(reportData -> { if (reportData.isPresent()) { @@ -702,29 +699,19 @@ public CompletableFuture createLookupResult(String candidateBroker } public boolean isBrokerActive(String candidateBroker) { - String candidateBrokerHostAndPort = parseHostAndPort(candidateBroker); Set availableBrokers = getAvailableBrokers(); - if (availableBrokers.contains(candidateBrokerHostAndPort)) { + if (availableBrokers.contains(candidateBroker)) { if (LOG.isDebugEnabled()) { - LOG.debug("Broker {} ({}) is available for.", candidateBroker, candidateBrokerHostAndPort); + LOG.debug("Broker {} is available for.", candidateBroker); } return true; } else { - LOG.warn("Broker {} ({}) couldn't be found in available brokers {}", - candidateBroker, candidateBrokerHostAndPort, - String.join(",", availableBrokers)); + LOG.warn("Broker {} couldn't be found in available brokers {}", + candidateBroker, String.join(",", availableBrokers)); return false; } } - private static String parseHostAndPort(String candidateBroker) { - int uriSeparatorPos = candidateBroker.indexOf("://"); - if (uriSeparatorPos == -1) { - throw new IllegalArgumentException("'" + candidateBroker + "' isn't an URI."); - } - return candidateBroker.substring(uriSeparatorPos + 3); - } - private Set getAvailableBrokers() { try { return loadManager.get().getAvailableBrokers(); @@ -740,7 +727,7 @@ private Set getAvailableBrokers() { * @return the least loaded broker addresses * @throws Exception if an error occurs */ - private Optional> getLeastLoadedFromLoadManager(ServiceUnitId serviceUnit) throws Exception { + private Optional getLeastLoadedFromLoadManager(ServiceUnitId serviceUnit) throws Exception { Optional leastLoadedBroker = loadManager.get().getLeastLoaded(serviceUnit); if (leastLoadedBroker.isEmpty()) { LOG.warn("No broker is available for {}", serviceUnit); @@ -748,15 +735,13 @@ private Optional> getLeastLoadedFromLoadManager(ServiceUnit } String lookupAddress = leastLoadedBroker.get().getResourceId(); - String advertisedAddr = (String) leastLoadedBroker.get() - .getProperty(ResourceUnit.PROPERTY_KEY_BROKER_ZNODE_NAME); if (LOG.isDebugEnabled()) { LOG.debug("{} : redirecting to the least loaded broker, lookup address={}", - pulsar.getSafeWebServiceAddress(), + pulsar.getBrokerId(), lookupAddress); } - return Optional.of(Pair.of(lookupAddress, advertisedAddr)); + return Optional.of(lookupAddress); } public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle) { @@ -1447,6 +1432,9 @@ private CompletableFuture> getPartitionsForTopic(TopicName topicNam }); } + /*** + * List persistent topics names under a namespace, the topic name contains the partition suffix. + */ public CompletableFuture> getListOfPersistentTopics(NamespaceName namespaceName) { return pulsar.getPulsarResources().getTopicResources().listPersistentTopicsAsync(namespaceName); } @@ -1564,7 +1552,7 @@ public CompletableFuture checkOwnershipPresentAsync(NamespaceBundle bun } public void unloadSLANamespace() throws Exception { - NamespaceName namespaceName = getSLAMonitorNamespace(pulsar.getLookupServiceAddress(), config); + NamespaceName namespaceName = getSLAMonitorNamespace(pulsar.getBrokerId(), config); LOG.info("Checking owner for SLA namespace {}", namespaceName); @@ -1597,7 +1585,7 @@ public static String checkHeartbeatNamespace(ServiceUnitId ns) { Matcher m = HEARTBEAT_NAMESPACE_PATTERN.matcher(ns.getNamespaceObject().toString()); if (m.matches()) { LOG.debug("Heartbeat namespace matched the lookup namespace {}", ns.getNamespaceObject().toString()); - return String.format("http://%s", m.group(1)); + return m.group(1); } else { return null; } @@ -1607,7 +1595,7 @@ public static String checkHeartbeatNamespaceV2(ServiceUnitId ns) { Matcher m = HEARTBEAT_NAMESPACE_PATTERN_V2.matcher(ns.getNamespaceObject().toString()); if (m.matches()) { LOG.debug("Heartbeat namespace v2 matched the lookup namespace {}", ns.getNamespaceObject().toString()); - return String.format("http://%s", m.group(1)); + return m.group(1); } else { return null; } @@ -1616,7 +1604,7 @@ public static String checkHeartbeatNamespaceV2(ServiceUnitId ns) { public static String getSLAMonitorBrokerName(ServiceUnitId ns) { Matcher m = SLA_NAMESPACE_PATTERN.matcher(ns.getNamespaceObject().toString()); if (m.matches()) { - return String.format("http://%s", m.group(1)); + return m.group(1); } else { return null; } @@ -1647,16 +1635,16 @@ public static boolean isHeartbeatNamespace(ServiceUnitId ns) { } public boolean registerSLANamespace() throws PulsarServerException { - String lookupServiceAddress = pulsar.getLookupServiceAddress(); - boolean isNameSpaceRegistered = registerNamespace(getSLAMonitorNamespace(lookupServiceAddress, config), false); + String brokerId = pulsar.getBrokerId(); + boolean isNameSpaceRegistered = registerNamespace(getSLAMonitorNamespace(brokerId, config), false); if (isNameSpaceRegistered) { if (LOG.isDebugEnabled()) { LOG.debug("Added SLA Monitoring namespace name in local cache: ns={}", - getSLAMonitorNamespace(lookupServiceAddress, config)); + getSLAMonitorNamespace(brokerId, config)); } } else if (LOG.isDebugEnabled()) { LOG.debug("SLA Monitoring not owned by the broker: ns={}", - getSLAMonitorNamespace(lookupServiceAddress, config)); + getSLAMonitorNamespace(brokerId, config)); } return isNameSpaceRegistered; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/protocol/ProtocolHandlers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/protocol/ProtocolHandlers.java index 42a82b2de762b..4059ccf5f26eb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/protocol/ProtocolHandlers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/protocol/ProtocolHandlers.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketAddress; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -51,6 +52,9 @@ public class ProtocolHandlers implements AutoCloseable { * @return the collection of protocol handlers */ public static ProtocolHandlers load(ServiceConfiguration conf) throws IOException { + if (conf.getMessagingProtocols().isEmpty()) { + return new ProtocolHandlers(Collections.emptyMap()); + } ProtocolHandlerDefinitions definitions = ProtocolHandlerUtils.searchForHandlers( conf.getProtocolHandlerDirectory(), conf.getNarExtractionDirectory()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java index 6dd296d16b53b..1b5b2824257b0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java @@ -194,7 +194,7 @@ protected synchronized CompletableFuture closeProducerAsync() { return CompletableFuture.completedFuture(null); } CompletableFuture future = producer.closeAsync(); - future.thenRun(() -> { + return future.thenRun(() -> { STATE_UPDATER.set(this, State.Stopped); this.producer = null; // deactivate further read @@ -209,7 +209,6 @@ protected synchronized CompletableFuture closeProducerAsync() { brokerService.executor().schedule(this::closeProducerAsync, waitTimeMs, TimeUnit.MILLISECONDS); return null; }); - return future; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 4077762bb0640..6363e197616ca 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -123,8 +123,6 @@ import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.service.persistent.SystemTopic; import org.apache.pulsar.broker.service.plugin.EntryFilterProvider; -import org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage; -import org.apache.pulsar.broker.service.schema.SchemaRegistryService; import org.apache.pulsar.broker.stats.ClusterReplicationMetrics; import org.apache.pulsar.broker.stats.prometheus.metrics.ObserverGauge; import org.apache.pulsar.broker.stats.prometheus.metrics.Summary; @@ -375,8 +373,8 @@ public BrokerService(PulsarService pulsar, EventLoopGroup eventLoopGroup) throws if (pulsar.getConfiguration().getMaxUnackedMessagesPerBroker() > 0 && pulsar.getConfiguration().getMaxUnackedMessagesPerSubscriptionOnBrokerBlocked() > 0.0) { this.maxUnackedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerBroker(); - this.maxUnackedMsgsPerDispatcher = (int) ((maxUnackedMessages - * pulsar.getConfiguration().getMaxUnackedMessagesPerSubscriptionOnBrokerBlocked()) / 100); + this.maxUnackedMsgsPerDispatcher = (int) (maxUnackedMessages + * pulsar.getConfiguration().getMaxUnackedMessagesPerSubscriptionOnBrokerBlocked()); log.info("Enabling per-broker unack-message limit {} and dispatcher-limit {} on blocked-broker", maxUnackedMessages, maxUnackedMsgsPerDispatcher); // block misbehaving dispatcher by checking periodically @@ -961,43 +959,49 @@ public CompletableFuture> getTopic(final TopicName topicName, bo } final boolean isPersistentTopic = topicName.getDomain().equals(TopicDomain.persistent); if (isPersistentTopic) { - final CompletableFuture> topicPoliciesFuture = - getTopicPoliciesBypassSystemTopic(topicName); - return topicPoliciesFuture.exceptionally(ex -> { - final Throwable rc = FutureUtil.unwrapCompletionException(ex); - final String errorInfo = String.format("Topic creation encountered an exception by initialize" - + " topic policies service. topic_name=%s error_message=%s", topicName, rc.getMessage()); - log.error(errorInfo, rc); - throw FutureUtil.wrapToCompletionException(new ServiceUnitNotReadyException(errorInfo)); - }).thenCompose(optionalTopicPolicies -> { - final TopicPolicies topicPolicies = optionalTopicPolicies.orElse(null); - return topics.computeIfAbsent(topicName.toString(), (tpName) -> { - if (topicName.isPartitioned()) { - final TopicName topicNameEntity = TopicName.get(topicName.getPartitionedTopicName()); - return fetchPartitionedTopicMetadataAsync(topicNameEntity) - .thenCompose((metadata) -> { - // Allow crate non-partitioned persistent topic that name includes `partition` - if (metadata.partitions == 0 - || topicName.getPartitionIndex() < metadata.partitions) { - return loadOrCreatePersistentTopic(tpName, createIfMissing, - properties, topicPolicies); - } - final String errorMsg = - String.format("Illegal topic partition name %s with max allowed " - + "%d partitions", topicName, metadata.partitions); - log.warn(errorMsg); - return FutureUtil - .failedFuture(new BrokerServiceException.NotAllowedException(errorMsg)); - }); - } - return loadOrCreatePersistentTopic(tpName, createIfMissing, properties, topicPolicies); - }).thenCompose(optionalTopic -> { - if (!optionalTopic.isPresent() && createIfMissing) { - log.warn("[{}] Try to recreate the topic with createIfMissing=true " - + "but the returned topic is empty", topicName); - return getTopic(topicName, createIfMissing, properties); - } - return CompletableFuture.completedFuture(optionalTopic); + return pulsar.getPulsarResources().getTopicResources().persistentTopicExists(topicName) + .thenCompose(exists -> { + if (!exists && !createIfMissing) { + return CompletableFuture.completedFuture(Optional.empty()); + } + return getTopicPoliciesBypassSystemTopic(topicName).exceptionally(ex -> { + final Throwable rc = FutureUtil.unwrapCompletionException(ex); + final String errorInfo = String.format("Topic creation encountered an exception by initialize" + + " topic policies service. topic_name=%s error_message=%s", topicName, + rc.getMessage()); + log.error(errorInfo, rc); + throw FutureUtil.wrapToCompletionException(new ServiceUnitNotReadyException(errorInfo)); + }).thenCompose(optionalTopicPolicies -> { + final TopicPolicies topicPolicies = optionalTopicPolicies.orElse(null); + return topics.computeIfAbsent(topicName.toString(), (tpName) -> { + if (topicName.isPartitioned()) { + final TopicName topicNameEntity = TopicName.get(topicName.getPartitionedTopicName()); + return fetchPartitionedTopicMetadataAsync(topicNameEntity) + .thenCompose((metadata) -> { + // Allow crate non-partitioned persistent topic that name includes + // `partition` + if (metadata.partitions == 0 + || topicName.getPartitionIndex() < metadata.partitions) { + return loadOrCreatePersistentTopic(tpName, createIfMissing, + properties, topicPolicies); + } + final String errorMsg = + String.format("Illegal topic partition name %s with max allowed " + + "%d partitions", topicName, metadata.partitions); + log.warn(errorMsg); + return FutureUtil.failedFuture( + new BrokerServiceException.NotAllowedException(errorMsg)); + }); + } + return loadOrCreatePersistentTopic(tpName, createIfMissing, properties, topicPolicies); + }).thenCompose(optionalTopic -> { + if (!optionalTopic.isPresent() && createIfMissing) { + log.warn("[{}] Try to recreate the topic with createIfMissing=true " + + "but the returned topic is empty", topicName); + return getTopic(topicName, createIfMissing, properties); + } + return CompletableFuture.completedFuture(optionalTopic); + }); }); }); } else { @@ -1773,10 +1777,17 @@ private CompletableFuture getManagedLedgerConfig(@Nonnull T } if (retentionPolicies == null) { - retentionPolicies = policies.map(p -> p.retention_policies).orElseGet( - () -> new RetentionPolicies(serviceConfig.getDefaultRetentionTimeInMinutes(), - serviceConfig.getDefaultRetentionSizeInMB()) - ); + if (SystemTopicNames.isSystemTopic(topicName)) { + if (log.isDebugEnabled()) { + log.debug("{} Disable data retention policy for system topic.", topicName); + } + retentionPolicies = new RetentionPolicies(0, 0); + } else { + retentionPolicies = policies.map(p -> p.retention_policies).orElseGet( + () -> new RetentionPolicies(serviceConfig.getDefaultRetentionTimeInMinutes(), + serviceConfig.getDefaultRetentionSizeInMB()) + ); + } } ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); @@ -2112,7 +2123,7 @@ public CompletableFuture checkTopicNsOwnership(final String topic) { } else { String msg = String.format("Namespace bundle for topic (%s) not served by this instance:%s. " + "Please redo the lookup. Request is denied: namespace=%s", - topic, pulsar.getLookupServiceAddress(), topicName.getNamespace()); + topic, pulsar.getBrokerId(), topicName.getNamespace()); log.warn(msg); return FutureUtil.failedFuture(new ServiceUnitNotReadyException(msg)); } @@ -3447,22 +3458,21 @@ public CompletableFuture deleteTopicPolicies(TopicName topicName) { } public CompletableFuture deleteSchema(TopicName topicName) { + // delete schema at the upper level when deleting the partitioned topic. + if (topicName.isPartitioned()) { + return CompletableFuture.completedFuture(null); + } String base = topicName.getPartitionedTopicName(); String id = TopicName.get(base).getSchemaName(); - SchemaRegistryService schemaRegistryService = getPulsar().getSchemaRegistryService(); - return BookkeeperSchemaStorage.ignoreUnrecoverableBKException(schemaRegistryService.getSchema(id)) - .thenCompose(schema -> { - if (schema != null) { - // It's different from `SchemasResource.deleteSchema` - // because when we delete a topic, the schema - // history is meaningless. But when we delete a schema of a topic, a new schema could be - // registered in the future. - log.info("Delete schema storage of id: {}", id); - return getPulsar().getSchemaRegistryService().deleteSchemaStorage(id); - } else { - return CompletableFuture.completedFuture(null); - } - }); + return getPulsar().getSchemaRegistryService().deleteSchemaStorage(id).whenComplete((vid, ex) -> { + if (vid != null && ex == null) { + // It's different from `SchemasResource.deleteSchema` + // because when we delete a topic, the schema + // history is meaningless. But when we delete a schema of a topic, a new schema could be + // registered in the future. + log.info("Deleted schema storage of id: {}", id); + } + }); } private CompletableFuture checkMaxTopicsPerNamespace(TopicName topicName, int numPartitions) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java index ea491bd40d332..b2b2b512c8cfc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java @@ -39,7 +39,8 @@ * number of keys assigned to each consumer. */ public class ConsistentHashingStickyKeyConsumerSelector implements StickyKeyConsumerSelector { - + // use NUL character as field separator for hash key calculation + private static final String KEY_SEPARATOR = "\0"; private final ReadWriteLock rwLock = new ReentrantReadWriteLock(); // Consistent-Hash ring @@ -59,8 +60,7 @@ public CompletableFuture addConsumer(Consumer consumer) { // Insert multiple points on the hash ring for every consumer // The points are deterministically added based on the hash of the consumer name for (int i = 0; i < numberOfPoints; i++) { - String key = consumer.consumerName() + i; - int hash = Murmur3_32Hash.getInstance().makeHash(key.getBytes()); + int hash = calculateHashForConsumerAndIndex(consumer, i); hashRing.compute(hash, (k, v) -> { if (v == null) { return Lists.newArrayList(consumer); @@ -79,14 +79,18 @@ public CompletableFuture addConsumer(Consumer consumer) { } } + private static int calculateHashForConsumerAndIndex(Consumer consumer, int index) { + String key = consumer.consumerName() + KEY_SEPARATOR + index; + return Murmur3_32Hash.getInstance().makeHash(key.getBytes()); + } + @Override public void removeConsumer(Consumer consumer) { rwLock.writeLock().lock(); try { // Remove all the points that were added for this consumer for (int i = 0; i < numberOfPoints; i++) { - String key = consumer.consumerName() + i; - int hash = Murmur3_32Hash.getInstance().makeHash(key.getBytes()); + int hash = calculateHashForConsumerAndIndex(consumer, i); hashRing.compute(hash, (k, v) -> { if (v == null) { return null; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 83dcd8d6c1616..4cd54420200be 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -324,7 +324,7 @@ public Future sendMessages(final List entries, EntryBatch if (pendingAcks != null) { int batchSize = batchSizes.getBatchSize(i); int stickyKeyHash = getStickyKeyHash(entry); - long[] ackSet = getCursorAckSet(PositionImpl.get(entry.getLedgerId(), entry.getEntryId())); + long[] ackSet = batchIndexesAcks == null ? null : batchIndexesAcks.getAckSet(i); if (ackSet != null) { unackedMessages -= (batchSize - BitSet.valueOf(ackSet).cardinality()); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSenderImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSenderImpl.java index dd74fc4e71ed2..105650caaaf13 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSenderImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSenderImpl.java @@ -356,12 +356,18 @@ public void sendEndTxnErrorResponse(long requestId, TxnID txnID, ServerError err writeAndFlush(outBuf); } + /*** + * @param topics topic names which are matching, the topic name contains the partition suffix. + */ @Override public void sendWatchTopicListSuccess(long requestId, long watcherId, String topicsHash, List topics) { BaseCommand command = Commands.newWatchTopicListSuccess(requestId, watcherId, topicsHash, topics); interceptAndWriteCommand(command); } + /*** + * {@inheritDoc} + */ @Override public void sendWatchTopicListUpdate(long watcherId, List newTopics, List deletedTopics, String topicsHash) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 9f2b98aeb40d9..0d9b5ea73e0c9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -2170,7 +2170,8 @@ protected void handleGetLastMessageId(CommandGetLastMessageId getLastMessageId) (PositionImpl) markDeletePosition, partitionIndex, requestId, - consumer.getSubscription().getName()); + consumer.getSubscription().getName(), + consumer.readCompacted()); }).exceptionally(e -> { writeAndFlush(Commands.newError(getLastMessageId.getRequestId(), ServerError.UnknownError, "Failed to recover Transaction Buffer.")); @@ -2188,15 +2189,17 @@ private void getLargestBatchIndexWhenPossible( PositionImpl markDeletePosition, int partitionIndex, long requestId, - String subscriptionName) { + String subscriptionName, + boolean readCompacted) { PersistentTopic persistentTopic = (PersistentTopic) topic; ManagedLedgerImpl ml = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); // If it's not pointing to a valid entry, respond messageId of the current position. // If the compaction cursor reach the end of the topic, respond messageId from compacted ledger - CompletableFuture compactionHorizonFuture = - persistentTopic.getTopicCompactionService().getLastCompactedPosition(); + CompletableFuture compactionHorizonFuture = readCompacted + ? persistentTopic.getTopicCompactionService().getLastCompactedPosition() : + CompletableFuture.completedFuture(null); compactionHorizonFuture.whenComplete((compactionHorizon, ex) -> { if (ex != null) { @@ -2205,8 +2208,22 @@ private void getLargestBatchIndexWhenPossible( return; } - if (lastPosition.getEntryId() == -1 || (compactionHorizon != null - && lastPosition.compareTo((PositionImpl) compactionHorizon) <= 0)) { + if (lastPosition.getEntryId() == -1 || !ml.ledgerExists(lastPosition.getLedgerId())) { + // there is no entry in the original topic + if (compactionHorizon != null) { + // if readCompacted is true, we need to read the last entry from compacted topic + handleLastMessageIdFromCompactionService(persistentTopic, requestId, partitionIndex, + markDeletePosition); + } else { + // if readCompacted is false, we need to return MessageId.earliest + writeAndFlush(Commands.newGetLastMessageIdResponse(requestId, -1, -1, partitionIndex, -1, + markDeletePosition != null ? markDeletePosition.getLedgerId() : -1, + markDeletePosition != null ? markDeletePosition.getEntryId() : -1)); + } + return; + } + + if (compactionHorizon != null && lastPosition.compareTo((PositionImpl) compactionHorizon) <= 0) { handleLastMessageIdFromCompactionService(persistentTopic, requestId, partitionIndex, markDeletePosition); return; @@ -2241,7 +2258,8 @@ public String toString() { batchSizeFuture.whenComplete((batchSize, e) -> { if (e != null) { - if (e.getCause() instanceof ManagedLedgerException.NonRecoverableLedgerException) { + if (e.getCause() instanceof ManagedLedgerException.NonRecoverableLedgerException + && readCompacted) { handleLastMessageIdFromCompactionService(persistentTopic, requestId, partitionIndex, markDeletePosition); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 80fecbe67b646..71f78e21f938f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -19,6 +19,8 @@ package org.apache.pulsar.broker.service; import static java.util.Objects.requireNonNull; +import com.github.benmanes.caffeine.cache.AsyncLoadingCache; +import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; import java.util.HashSet; @@ -29,6 +31,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.Nonnull; import org.apache.commons.lang3.tuple.MutablePair; @@ -84,10 +87,25 @@ public class SystemTopicBasedTopicPoliciesService implements TopicPoliciesServic @VisibleForTesting final Map>> listeners = new ConcurrentHashMap<>(); + private final AsyncLoadingCache> writerCaches; + public SystemTopicBasedTopicPoliciesService(PulsarService pulsarService) { this.pulsarService = pulsarService; this.clusterName = pulsarService.getConfiguration().getClusterName(); this.localCluster = Sets.newHashSet(clusterName); + this.writerCaches = Caffeine.newBuilder() + .expireAfterAccess(5, TimeUnit.MINUTES) + .removalListener((namespaceName, writer, cause) -> { + ((SystemTopicClient.Writer) writer).closeAsync().exceptionally(ex -> { + log.error("[{}] Close writer error.", namespaceName, ex); + return null; + }); + }) + .buildAsync((namespaceName, executor) -> { + SystemTopicClient systemTopicClient = namespaceEventsSystemTopicFactory + .createTopicPoliciesSystemTopicClient(namespaceName); + return systemTopicClient.newWriterAsync(); + }); } @Override @@ -122,39 +140,32 @@ private CompletableFuture sendTopicPolicyEvent(TopicName topicName, Action } catch (PulsarServerException e) { return CompletableFuture.failedFuture(e); } - - SystemTopicClient systemTopicClient = namespaceEventsSystemTopicFactory - .createTopicPoliciesSystemTopicClient(topicName.getNamespaceObject()); - - return systemTopicClient.newWriterAsync() - .thenCompose(writer -> { - PulsarEvent event = getPulsarEvent(topicName, actionType, policies); - CompletableFuture writeFuture = - ActionType.DELETE.equals(actionType) ? writer.deleteAsync(getEventKey(event), event) - : writer.writeAsync(getEventKey(event), event); - return writeFuture.handle((messageId, e) -> { - if (e != null) { - return CompletableFuture.failedFuture(e); + CompletableFuture result = new CompletableFuture<>(); + writerCaches.get(topicName.getNamespaceObject()) + .whenComplete((writer, cause) -> { + if (cause != null) { + writerCaches.synchronous().invalidate(topicName.getNamespaceObject()); + result.completeExceptionally(cause); } else { - if (messageId != null) { - return CompletableFuture.completedFuture(null); - } else { - return CompletableFuture.failedFuture( - new RuntimeException("Got message id is null.")); - } - } - }).thenRun(() -> - writer.closeAsync().whenComplete((v, cause) -> { - if (cause != null) { - log.error("[{}] Close writer error.", topicName, cause); + PulsarEvent event = getPulsarEvent(topicName, actionType, policies); + CompletableFuture writeFuture = ActionType.DELETE.equals(actionType) + ? writer.deleteAsync(getEventKey(event), event) + : writer.writeAsync(getEventKey(event), event); + writeFuture.whenComplete((messageId, e) -> { + if (e != null) { + result.completeExceptionally(e); + } else { + if (messageId != null) { + result.complete(null); } else { - if (log.isDebugEnabled()) { - log.debug("[{}] Close writer success.", topicName); - } + result.completeExceptionally( + new RuntimeException("Got message id is null.")); } - }) - ); + } + }); + } }); + return result; }); } @@ -364,7 +375,7 @@ public CompletableFuture removeOwnedNamespaceBundleAsync(NamespaceBundle n } AtomicInteger bundlesCount = ownedBundlesCountPerNamespace.get(namespace); if (bundlesCount == null || bundlesCount.decrementAndGet() <= 0) { - cleanCacheAndCloseReader(namespace, true); + cleanCacheAndCloseReader(namespace, true, true); } return CompletableFuture.completedFuture(null); } @@ -440,6 +451,14 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp } private void cleanCacheAndCloseReader(@Nonnull NamespaceName namespace, boolean cleanOwnedBundlesCount) { + cleanCacheAndCloseReader(namespace, cleanOwnedBundlesCount, false); + } + + private void cleanCacheAndCloseReader(@Nonnull NamespaceName namespace, boolean cleanOwnedBundlesCount, + boolean cleanWriterCache) { + if (cleanWriterCache) { + writerCaches.synchronous().invalidate(namespace); + } CompletableFuture> readerFuture = readerCaches.remove(namespace); if (cleanOwnedBundlesCount) { @@ -688,5 +707,10 @@ protected Map>> getListeners( return listeners; } + @VisibleForTesting + protected AsyncLoadingCache> getWriterCaches() { + return writerCaches; + } + private static final Logger log = LoggerFactory.getLogger(SystemTopicBasedTopicPoliciesService.class); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java index 7aa50057d73c9..aea5b9fc65b46 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java @@ -31,6 +31,7 @@ import org.apache.pulsar.common.api.proto.CommandWatchTopicListClose; import org.apache.pulsar.common.api.proto.ServerError; import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.topics.TopicList; import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap; import org.apache.pulsar.metadata.api.NotificationType; @@ -42,11 +43,16 @@ public class TopicListService { public static class TopicListWatcher implements BiConsumer { + /** Topic names which are matching, the topic name contains the partition suffix. **/ private final List matchingTopics; private final TopicListService topicListService; private final long id; + /** The regexp for the topic name(not contains partition suffix). **/ private final Pattern topicsPattern; + /*** + * @param topicsPattern The regexp for the topic name(not contains partition suffix). + */ public TopicListWatcher(TopicListService topicListService, long id, Pattern topicsPattern, List topics) { this.topicListService = topicListService; @@ -59,9 +65,12 @@ public List getMatchingTopics() { return matchingTopics; } + /*** + * @param topicName topic name which contains partition suffix. + */ @Override public void accept(String topicName, NotificationType notificationType) { - if (topicsPattern.matcher(topicName).matches()) { + if (topicsPattern.matcher(TopicName.get(topicName).getPartitionedTopicName()).matches()) { List newTopics; List deletedTopics; if (notificationType == NotificationType.Deleted) { @@ -109,6 +118,9 @@ public void inactivate() { } } + /*** + * @param topicsPattern The regexp for the topic name(not contains partition suffix). + */ public void handleWatchTopicList(NamespaceName namespaceName, long watcherId, long requestId, Pattern topicsPattern, String topicsHash, Semaphore lookupSemaphore) { @@ -184,7 +196,9 @@ public void handleWatchTopicList(NamespaceName namespaceName, long watcherId, lo }); } - + /*** + * @param topicsPattern The regexp for the topic name(not contains partition suffix). + */ public void initializeTopicsListWatcher(CompletableFuture watcherFuture, NamespaceName namespace, long watcherId, Pattern topicsPattern) { namespaceService.getListOfPersistentTopics(namespace). @@ -246,6 +260,10 @@ public void deleteTopicListWatcher(Long watcherId) { log.info("[{}] Closed watcher, watcherId={}", connection.getRemoteAddress(), watcherId); } + /** + * @param deletedTopics topic names deleted(contains the partition suffix). + * @param newTopics topics names added(contains the partition suffix). + */ public void sendTopicListUpdate(long watcherId, String topicsHash, List deletedTopics, List newTopics) { connection.getCommandSender().sendWatchTopicListUpdate(watcherId, newTopics, deletedTopics, topicsHash); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java index 29bca715741ad..399a524a197e9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java @@ -191,7 +191,7 @@ public RedeliveryTracker getRedeliveryTracker() { } @Override - public void sendMessages(List entries) { + public synchronized void sendMessages(List entries) { Consumer consumer = TOTAL_AVAILABLE_PERMITS_UPDATER.get(this) > 0 ? getNextConsumer() : null; if (consumer != null) { SendMessageInfo sendMessageInfo = SendMessageInfo.getThreadLocal(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 00cf3a6583b9a..2fa85f262de37 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -968,7 +968,7 @@ public CompletableFuture asyncGetStats(GetStatsOptions }); stats.topicEpoch = topicEpoch.orElse(null); - stats.ownerBroker = brokerService.pulsar().getLookupServiceAddress(); + stats.ownerBroker = brokerService.pulsar().getBrokerId(); future.complete(stats); return future; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java index 191b19e317163..4ef2710fea61a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java @@ -123,18 +123,6 @@ protected boolean replicateEntries(List entries) { continue; } - if (msg.isExpired(messageTTLInSeconds)) { - msgExpired.recordEvent(0 /* no value stat */); - if (log.isDebugEnabled()) { - log.debug("[{}] Discarding expired message at position {}, replicateTo {}", - replicatorId, entry.getPosition(), msg.getReplicateTo()); - } - cursor.asyncDelete(entry.getPosition(), this, entry.getPosition()); - entry.release(); - msg.recycle(); - continue; - } - if (STATE_UPDATER.get(this) != State.Started || isLocalMessageSkippedOnce) { // The producer is not ready yet after having stopped/restarted. Drop the message because it will // recovered when the producer is ready diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java index 806773af45189..387ba83d9cd62 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java @@ -54,6 +54,7 @@ import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.compaction.CompactedTopicUtils; +import org.apache.pulsar.compaction.Compactor; import org.apache.pulsar.compaction.TopicCompactionService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -107,9 +108,9 @@ protected void scheduleReadOnActiveConsumer() { if (log.isDebugEnabled()) { log.debug("[{}] Rewind cursor and read more entries without delay", name); } - cursor.rewind(); - Consumer activeConsumer = ACTIVE_CONSUMER_UPDATER.get(this); + cursor.rewind(activeConsumer != null && activeConsumer.readCompacted()); + notifyActiveConsumerChanged(activeConsumer); readMoreEntries(activeConsumer); return; @@ -127,9 +128,9 @@ protected void scheduleReadOnActiveConsumer() { log.debug("[{}] Rewind cursor and read more entries after {} ms delay", name, serviceConfig.getActiveConsumerFailoverDelayTimeMillis()); } - cursor.rewind(); - Consumer activeConsumer = ACTIVE_CONSUMER_UPDATER.get(this); + cursor.rewind(activeConsumer != null && activeConsumer.readCompacted()); + notifyActiveConsumerChanged(activeConsumer); readMoreEntries(activeConsumer); readOnActiveConsumerTask = null; @@ -206,7 +207,7 @@ private synchronized void internalReadEntriesComplete(final List entries, } } entries.forEach(Entry::release); - cursor.rewind(); + cursor.rewind(currentConsumer != null ? currentConsumer.readCompacted() : readConsumer.readCompacted()); if (currentConsumer != null) { notifyActiveConsumerChanged(currentConsumer); readMoreEntries(currentConsumer); @@ -301,7 +302,7 @@ private synchronized void internalRedeliverUnacknowledgedMessages(Consumer consu } cursor.cancelPendingReadRequest(); havePendingRead = false; - cursor.rewind(); + cursor.rewind(consumer.readCompacted()); if (log.isDebugEnabled()) { log.debug("[{}-{}] Cursor rewinded, redelivering unacknowledged messages. ", name, consumer); } @@ -360,7 +361,9 @@ private void readMoreEntries(Consumer consumer) { } havePendingRead = true; if (consumer.readCompacted()) { - boolean readFromEarliest = isFirstRead && MessageId.earliest.equals(consumer.getStartMessageId()); + boolean readFromEarliest = isFirstRead && MessageId.earliest.equals(consumer.getStartMessageId()) + && (!cursor.isDurable() || cursor.getName().equals(Compactor.COMPACTION_SUBSCRIPTION) + || hasValidMarkDeletePosition(cursor)); TopicCompactionService topicCompactionService = topic.getTopicCompactionService(); CompactedTopicUtils.asyncReadCompactedEntries(topicCompactionService, cursor, messagesToRead, bytesToRead, topic.getMaxReadPosition(), readFromEarliest, this, true, consumer); @@ -378,6 +381,13 @@ private void readMoreEntries(Consumer consumer) { } } + private boolean hasValidMarkDeletePosition(ManagedCursor cursor) { + // If `markDeletedPosition.entryID == -1L` then the md-position is an invalid position, + // since the initial md-position of the consumer will be set to it. + // See ManagedLedgerImpl#asyncOpenCursor and ManagedLedgerImpl#getFirstPosition + return cursor.getMarkDeletedPosition() != null && cursor.getMarkDeletedPosition().getEntryId() == -1L; + } + @Override protected void reScheduleRead() { if (isRescheduleReadInProgress.compareAndSet(false, true)) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java index 020dc5323e55b..ac391c1050340 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java @@ -18,8 +18,10 @@ */ package org.apache.pulsar.broker.service.persistent; +import com.google.common.annotations.VisibleForTesting; import java.util.Objects; import java.util.Optional; +import java.util.SortedMap; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.LongAdder; import javax.annotation.Nullable; @@ -30,8 +32,10 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.LedgerNotExistException; import org.apache.bookkeeper.mledger.ManagedLedgerException.NonRecoverableLedgerException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.pulsar.broker.service.MessageExpirer; import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; @@ -48,7 +52,6 @@ public class PersistentMessageExpiryMonitor implements FindEntryCallback, Messag private final String topicName; private final Rate msgExpired; private final LongAdder totalMsgExpired; - private final boolean autoSkipNonRecoverableData; private final PersistentSubscription subscription; private static final int FALSE = 0; @@ -68,8 +71,12 @@ public PersistentMessageExpiryMonitor(PersistentTopic topic, String subscription this.subscription = subscription; this.msgExpired = new Rate(); this.totalMsgExpired = new LongAdder(); + } + + @VisibleForTesting + public boolean isAutoSkipNonRecoverableData() { // check to avoid test failures - this.autoSkipNonRecoverableData = this.cursor.getManagedLedger() != null + return this.cursor.getManagedLedger() != null && this.cursor.getManagedLedger().getConfig().isAutoSkipNonRecoverableData(); } @@ -78,7 +85,9 @@ public boolean expireMessages(int messageTTLInSeconds) { if (expirationCheckInProgressUpdater.compareAndSet(this, FALSE, TRUE)) { log.info("[{}][{}] Starting message expiry check, ttl= {} seconds", topicName, subName, messageTTLInSeconds); - + // First filter the entire Ledger reached TTL based on the Ledger closing time to avoid client clock skew + checkExpiryByLedgerClosureTime(cursor, messageTTLInSeconds); + // Some part of entries in active Ledger may have reached TTL, so we need to continue searching. cursor.asyncFindNewestMatching(ManagedCursor.FindPositionConstraint.SearchActiveEntries, entry -> { try { long entryTimestamp = Commands.getEntryTimestamp(entry.getDataBuffer()); @@ -100,6 +109,35 @@ public boolean expireMessages(int messageTTLInSeconds) { } } + private void checkExpiryByLedgerClosureTime(ManagedCursor cursor, int messageTTLInSeconds) { + if (messageTTLInSeconds <= 0) { + return; + } + if (cursor instanceof ManagedCursorImpl managedCursor) { + ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) managedCursor.getManagedLedger(); + Position deletedPosition = managedCursor.getMarkDeletedPosition(); + SortedMap ledgerInfoSortedMap = + managedLedger.getLedgersInfo().subMap(deletedPosition.getLedgerId(), true, + managedLedger.getLedgersInfo().lastKey(), true); + MLDataFormats.ManagedLedgerInfo.LedgerInfo info = null; + for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ledgerInfo : ledgerInfoSortedMap.values()) { + if (!ledgerInfo.hasTimestamp() || !MessageImpl.isEntryExpired(messageTTLInSeconds, + ledgerInfo.getTimestamp())) { + break; + } + info = ledgerInfo; + } + if (info != null && info.getLedgerId() > -1) { + PositionImpl position = PositionImpl.get(info.getLedgerId(), info.getEntries() - 1); + if (((PositionImpl) managedLedger.getLastConfirmedEntry()).compareTo(position) < 0) { + findEntryComplete(managedLedger.getLastConfirmedEntry(), null); + } else { + findEntryComplete(position, null); + } + } + } + } + @Override public boolean expireMessages(Position messagePosition) { // If it's beyond last position of this topic, do nothing. @@ -177,7 +215,8 @@ public void findEntryComplete(Position position, Object ctx) { if (position != null) { log.info("[{}][{}] Expiring all messages until position {}", topicName, subName, position); Position prevMarkDeletePos = cursor.getMarkDeletedPosition(); - cursor.asyncMarkDelete(position, markDeleteCallback, cursor.getNumberOfEntriesInBacklog(false)); + cursor.asyncMarkDelete(position, cursor.getProperties(), markDeleteCallback, + cursor.getNumberOfEntriesInBacklog(false)); if (!Objects.equals(cursor.getMarkDeletedPosition(), prevMarkDeletePos) && subscription != null) { subscription.updateLastMarkDeleteAdvancedTimestamp(); } @@ -195,7 +234,7 @@ public void findEntryFailed(ManagedLedgerException exception, Optional if (log.isDebugEnabled()) { log.debug("[{}][{}] Finding expired entry operation failed", topicName, subName, exception); } - if (autoSkipNonRecoverableData && failedReadPosition.isPresent() + if (isAutoSkipNonRecoverableData() && failedReadPosition.isPresent() && (exception instanceof NonRecoverableLedgerException)) { log.warn("[{}][{}] read failed from ledger at position:{} : {}", topicName, subName, failedReadPosition, exception.getMessage()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index dc79146110f00..e5d90bf0ef42f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.Optional; import java.util.TreeMap; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; @@ -514,9 +515,15 @@ public String getTypeString() { return "Null"; } - @Override public CompletableFuture analyzeBacklog(Optional position) { - + final ManagedLedger managedLedger = topic.getManagedLedger(); + final String newNonDurableCursorName = "analyze-backlog-" + UUID.randomUUID(); + ManagedCursor newNonDurableCursor; + try { + newNonDurableCursor = ((ManagedCursorImpl) cursor).duplicateNonDurableCursor(newNonDurableCursorName); + } catch (ManagedLedgerException e) { + return CompletableFuture.failedFuture(e); + } long start = System.currentTimeMillis(); if (log.isDebugEnabled()) { log.debug("[{}][{}] Starting to analyze backlog", topicName, subName); @@ -531,7 +538,7 @@ public CompletableFuture analyzeBacklog(Optional AtomicLong rejectedMessages = new AtomicLong(); AtomicLong rescheduledMessages = new AtomicLong(); - Position currentPosition = cursor.getMarkDeletedPosition(); + Position currentPosition = newNonDurableCursor.getMarkDeletedPosition(); if (log.isDebugEnabled()) { log.debug("[{}][{}] currentPosition {}", @@ -591,7 +598,7 @@ public CompletableFuture analyzeBacklog(Optional return true; }; - return cursor.scan( + CompletableFuture res = newNonDurableCursor.scan( position, condition, batchSize, @@ -618,7 +625,22 @@ public CompletableFuture analyzeBacklog(Optional topicName, subName, end - start, result); return result; }); + res.whenComplete((__, ex) -> { + managedLedger.asyncDeleteCursor(newNonDurableCursorName, + new AsyncCallbacks.DeleteCursorCallback(){ + @Override + public void deleteCursorComplete(Object ctx) { + // Nothing to do. + } + @Override + public void deleteCursorFailed(ManagedLedgerException exception, Object ctx) { + log.warn("[{}][{}] Delete non-durable cursor[{}] failed when analyze backlog.", + topicName, subName, newNonDurableCursor.getName()); + } + }, null); + }); + return res; } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 48069cf555448..bb0796b48fd77 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -230,7 +230,8 @@ public static boolean isDedupCursorName(String name) { protected final MessageDeduplication messageDeduplication; private static final Long COMPACTION_NEVER_RUN = -0xfebecffeL; - private CompletableFuture currentCompaction = CompletableFuture.completedFuture(COMPACTION_NEVER_RUN); + private volatile CompletableFuture currentCompaction = CompletableFuture.completedFuture( + COMPACTION_NEVER_RUN); private TopicCompactionService topicCompactionService; // TODO: Create compaction strategy from topic policy when exposing strategic compaction to users. @@ -514,7 +515,7 @@ private PersistentSubscription createPersistentSubscription(String subscriptionN } } - private static boolean isCompactionSubscription(String subscriptionName) { + public static boolean isCompactionSubscription(String subscriptionName) { return COMPACTION_SUBSCRIPTION.equals(subscriptionName); } @@ -1006,7 +1007,9 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs } private CompletableFuture getDurableSubscription(String subscriptionName, - InitialPosition initialPosition, long startMessageRollbackDurationSec, boolean replicated, + InitialPosition initialPosition, + long startMessageRollbackDurationSec, + boolean replicated, Map subscriptionProperties) { CompletableFuture subscriptionFuture = new CompletableFuture<>(); if (checkMaxSubscriptionsPerTopicExceed(subscriptionName)) { @@ -1016,7 +1019,6 @@ private CompletableFuture getDurableSubscription(String subscripti } Map properties = PersistentSubscription.getBaseCursorProperties(replicated); - ledger.asyncOpenCursor(Codec.encode(subscriptionName), initialPosition, properties, subscriptionProperties, new OpenCursorCallback() { @Override @@ -1196,13 +1198,14 @@ private void asyncDeleteCursorWithClearDelayedMessage(String subscriptionName, CompletableFuture unsubscribeFuture) { PersistentSubscription persistentSubscription = subscriptions.get(subscriptionName); if (persistentSubscription == null) { - log.warn("[{}][{}] Can't find subscription, skip clear delayed message", topic, subscriptionName); + log.warn("[{}][{}] Can't find subscription, skip delete cursor", topic, subscriptionName); unsubscribeFuture.complete(null); return; } + if (!isDelayedDeliveryEnabled() || !(brokerService.getDelayedDeliveryTrackerFactory() instanceof BucketDelayedDeliveryTrackerFactory)) { - asyncDeleteCursor(subscriptionName, unsubscribeFuture); + asyncDeleteCursorWithCleanCompactionLedger(persistentSubscription, unsubscribeFuture); return; } @@ -1217,7 +1220,7 @@ private void asyncDeleteCursorWithClearDelayedMessage(String subscriptionName, if (ex != null) { unsubscribeFuture.completeExceptionally(ex); } else { - asyncDeleteCursor(subscriptionName, unsubscribeFuture); + asyncDeleteCursorWithCleanCompactionLedger(persistentSubscription, unsubscribeFuture); } }); } @@ -1227,6 +1230,29 @@ private void asyncDeleteCursorWithClearDelayedMessage(String subscriptionName, dispatcher.clearDelayedMessages().whenComplete((__, ex) -> { if (ex != null) { unsubscribeFuture.completeExceptionally(ex); + } else { + asyncDeleteCursorWithCleanCompactionLedger(persistentSubscription, unsubscribeFuture); + } + }); + } + + private void asyncDeleteCursorWithCleanCompactionLedger(PersistentSubscription subscription, + CompletableFuture unsubscribeFuture) { + final String subscriptionName = subscription.getName(); + if ((!isCompactionSubscription(subscriptionName)) || !(subscription instanceof PulsarCompactorSubscription)) { + asyncDeleteCursor(subscriptionName, unsubscribeFuture); + return; + } + + currentCompaction.handle((__, e) -> { + if (e != null) { + log.warn("[{}][{}] Last compaction task failed", topic, subscriptionName); + } + return ((PulsarCompactorSubscription) subscription).cleanCompactedLedger(); + }).whenComplete((__, ex) -> { + if (ex != null) { + log.error("[{}][{}] Error cleaning compacted ledger", topic, subscriptionName, ex); + unsubscribeFuture.completeExceptionally(ex); } else { asyncDeleteCursor(subscriptionName, unsubscribeFuture); } @@ -1729,11 +1755,11 @@ private CompletableFuture checkShadowReplication() { public void checkMessageExpiry() { int messageTtlInSeconds = topicPolicies.getMessageTTLInSeconds().get(); if (messageTtlInSeconds != 0) { - subscriptions.forEach((__, sub) -> sub.expireMessages(messageTtlInSeconds)); - replicators.forEach((__, replicator) - -> ((PersistentReplicator) replicator).expireMessages(messageTtlInSeconds)); - shadowReplicators.forEach((__, replicator) - -> ((PersistentReplicator) replicator).expireMessages(messageTtlInSeconds)); + subscriptions.forEach((__, sub) -> { + if (!isCompactionSubscription(sub.getName())) { + sub.expireMessages(messageTtlInSeconds); + } + }); } } @@ -2384,7 +2410,7 @@ public CompletableFuture asyncGetStats(GetStatsOptions stats.backlogSize = ledger.getEstimatedBacklogSize(); stats.deduplicationStatus = messageDeduplication.getStatus().toString(); stats.topicEpoch = topicEpoch.orElse(null); - stats.ownerBroker = brokerService.pulsar().getLookupServiceAddress(); + stats.ownerBroker = brokerService.pulsar().getBrokerId(); stats.offloadedStorageSize = ledger.getOffloadedSize(); stats.lastOffloadLedgerId = ledger.getLastOffloadedLedgerId(); stats.lastOffloadSuccessTimeStamp = ledger.getLastOffloadedSuccessTimestamp(); @@ -2892,7 +2918,7 @@ public void checkGC() { replCloseFuture.thenCompose(v -> delete(deleteMode == InactiveTopicDeleteMode.delete_when_no_subscriptions, deleteMode == InactiveTopicDeleteMode.delete_when_subscriptions_caught_up, false)) - .thenApply((res) -> tryToDeletePartitionedMetadata()) + .thenCompose((res) -> tryToDeletePartitionedMetadata()) .thenRun(() -> log.info("[{}] Topic deleted successfully due to inactivity", topic)) .exceptionally(e -> { if (e.getCause() instanceof TopicBusyException) { @@ -2900,6 +2926,8 @@ public void checkGC() { if (log.isDebugEnabled()) { log.debug("[{}] Did not delete busy topic: {}", topic, e.getCause().getMessage()); } + } else if (e.getCause() instanceof UnsupportedOperationException) { + log.info("[{}] Skip to delete partitioned topic: {}", topic, e.getCause().getMessage()); } else { log.warn("[{}] Inactive topic deletion failed", topic, e); } @@ -2944,7 +2972,7 @@ private CompletableFuture tryToDeletePartitionedMetadata() { .filter(topicExist -> topicExist) .findAny(); if (anyExistPartition.isPresent()) { - log.error("[{}] Delete topic metadata failed because" + log.info("[{}] Delete topic metadata failed because" + " another partition exist.", topicName); throw new UnsupportedOperationException( String.format("Another partition exists for [%s].", @@ -3427,17 +3455,29 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { public synchronized void triggerCompaction() throws PulsarServerException, AlreadyRunningException { if (currentCompaction.isDone()) { + if (!lock.readLock().tryLock()) { + log.info("[{}] Conflict topic-close, topic-delete, skip triggering compaction", topic); + return; + } + try { + if (isClosingOrDeleting) { + log.info("[{}] Topic is closing or deleting, skip triggering compaction", topic); + return; + } - if (strategicCompactionMap.containsKey(topic)) { - currentCompaction = brokerService.pulsar().getStrategicCompactor() - .compact(topic, strategicCompactionMap.get(topic)); - } else { - currentCompaction = topicCompactionService.compact().thenApply(x -> null); + if (strategicCompactionMap.containsKey(topic)) { + currentCompaction = brokerService.pulsar().getStrategicCompactor() + .compact(topic, strategicCompactionMap.get(topic)); + } else { + currentCompaction = topicCompactionService.compact().thenApply(x -> null); + } + } finally { + lock.readLock().unlock(); } currentCompaction.whenComplete((ignore, ex) -> { - if (ex != null){ - log.warn("[{}] Compaction failure.", topic, ex); - } + if (ex != null) { + log.warn("[{}] Compaction failure.", topic, ex); + } }); } else { throw new AlreadyRunningException("Compaction already in progress"); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PulsarCompactorSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PulsarCompactorSubscription.java index dbb09f6ac39fd..fe13aeb572e2e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PulsarCompactorSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PulsarCompactorSubscription.java @@ -22,12 +22,15 @@ import static org.apache.pulsar.broker.service.AbstractBaseDispatcher.checkAndApplyReachedEndOfTopicOrTopicMigration; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; import org.apache.pulsar.common.api.proto.CommandAck.AckType; import org.apache.pulsar.compaction.CompactedTopic; +import org.apache.pulsar.compaction.CompactedTopicContext; +import org.apache.pulsar.compaction.CompactedTopicImpl; import org.apache.pulsar.compaction.Compactor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -106,5 +109,19 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { } } + CompletableFuture cleanCompactedLedger() { + final CompletableFuture compactedTopicContextFuture = + ((CompactedTopicImpl) compactedTopic).getCompactedTopicContextFuture(); + if (compactedTopicContextFuture != null) { + return compactedTopicContextFuture.thenCompose(context -> { + long compactedLedgerId = context.getLedger().getId(); + ((CompactedTopicImpl) compactedTopic).reset(); + return compactedTopic.deleteCompactedLedger(compactedLedgerId); + }); + } else { + return CompletableFuture.completedFuture(null); + } + } + private static final Logger log = LoggerFactory.getLogger(PulsarCompactorSubscription.class); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java index 493072eb0c837..85e837ff1879a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java @@ -76,18 +76,6 @@ protected boolean replicateEntries(List entries) { continue; } - if (msg.isExpired(messageTTLInSeconds)) { - msgExpired.recordEvent(0 /* no value stat */); - if (log.isDebugEnabled()) { - log.debug("[{}] Discarding expired message at position {}, replicateTo {}", - replicatorId, entry.getPosition(), msg.getReplicateTo()); - } - cursor.asyncDelete(entry.getPosition(), this, entry.getPosition()); - entry.release(); - msg.recycle(); - continue; - } - if (STATE_UPDATER.get(this) != State.Started || isLocalMessageSkippedOnce) { // The producer is not ready yet after having stopped/restarted. Drop the message because it will // recovered when the producer is ready diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java index 78e30f6fff827..c509764bf6710 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java @@ -707,7 +707,8 @@ public static Exception bkException(String operation, int rc, long ledgerId, lon message += " - entry=" + entryId; } boolean recoverable = rc != BKException.Code.NoSuchLedgerExistsException - && rc != BKException.Code.NoSuchEntryException; + && rc != BKException.Code.NoSuchEntryException + && rc != BKException.Code.NoSuchLedgerExistsOnMetadataServerException; return new SchemaException(recoverable, message); } @@ -716,7 +717,8 @@ public static CompletableFuture ignoreUnrecoverableBKException(Completabl if (t.getCause() != null && (t.getCause() instanceof SchemaException) && !((SchemaException) t.getCause()).isRecoverable()) { - // Meeting NoSuchLedgerExistsException or NoSuchEntryException when reading schemas in + // Meeting NoSuchLedgerExistsException, NoSuchEntryException or + // NoSuchLedgerExistsOnMetadataServerException when reading schemas in // bookkeeper. This also means that the data has already been deleted by other operations // in deleting schema. if (log.isDebugEnabled()) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/DimensionStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/DimensionStats.java index 1b6f981ca4e21..54965e4c783d8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/DimensionStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/DimensionStats.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.stats; +import static com.google.common.base.Preconditions.checkArgument; import static io.prometheus.client.CollectorRegistry.defaultRegistry; import io.prometheus.client.Collector; import io.prometheus.client.Summary; @@ -70,6 +71,7 @@ public DimensionStats(String name, long updateDurationInSec) { } public void recordDimensionTimeValue(long latency, TimeUnit unit) { + checkArgument(latency >= 0); summary.observe(unit.toMillis(latency)); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java index 8ade2bc883f9a..c2816f5a2a013 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.stats.prometheus.metrics; +import static com.google.common.base.Preconditions.checkArgument; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; import org.apache.bookkeeper.stats.Counter; @@ -57,6 +58,7 @@ public void addCount(long delta) { @Override public void addLatency(long eventLatency, TimeUnit unit) { + checkArgument(eventLatency >= 0); long valueMillis = unit.toMillis(eventLatency); counter.add(valueMillis); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TopicPoliciesSystemTopicClient.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TopicPoliciesSystemTopicClient.java index 3fd8921c15efa..b7cff2e08c2d0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TopicPoliciesSystemTopicClient.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TopicPoliciesSystemTopicClient.java @@ -30,6 +30,8 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.TypedMessageBuilder; +import org.apache.pulsar.client.api.schema.SchemaDefinition; +import org.apache.pulsar.client.internal.DefaultImplementation; import org.apache.pulsar.common.events.ActionType; import org.apache.pulsar.common.events.PulsarEvent; import org.apache.pulsar.common.naming.TopicName; @@ -41,13 +43,17 @@ */ public class TopicPoliciesSystemTopicClient extends SystemTopicClientBase { + static Schema avroSchema = DefaultImplementation.getDefaultImplementation() + .newAvroSchema(SchemaDefinition.builder().withPojo(PulsarEvent.class).build()); + public TopicPoliciesSystemTopicClient(PulsarClient client, TopicName topicName) { super(client, topicName); + } @Override protected CompletableFuture> newWriterAsyncInternal() { - return client.newProducer(Schema.AVRO(PulsarEvent.class)) + return client.newProducer(avroSchema) .topic(topicName.toString()) .enableBatching(false) .createAsync() @@ -61,7 +67,7 @@ protected CompletableFuture> newWriterAsyncInternal() { @Override protected CompletableFuture> newReaderAsyncInternal() { - return client.newReader(Schema.AVRO(PulsarEvent.class)) + return client.newReader(avroSchema) .topic(topicName.toString()) .startMessageId(MessageId.earliest) .readCompacted(true) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index f356921d6988e..5392e473947e6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -287,8 +287,8 @@ private void handleTransactionMessage(TxnID txnId, Position position) { .checkAbortedTransaction(txnId)) { ongoingTxns.put(txnId, (PositionImpl) position); PositionImpl firstPosition = ongoingTxns.get(ongoingTxns.firstKey()); - //max read position is less than first ongoing transaction message position, so entryId -1 - maxReadPosition = PositionImpl.get(firstPosition.getLedgerId(), firstPosition.getEntryId() - 1); + // max read position is less than first ongoing transaction message position + maxReadPosition = ((ManagedLedgerImpl) topic.getManagedLedger()).getPreviousPosition(firstPosition); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index e8192cde3fdf3..e23286ae4492e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -56,7 +56,9 @@ import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.broker.authorization.AuthorizationService; +import org.apache.pulsar.broker.loadbalance.LoadManager; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.namespace.LookupOptions; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.resources.BookieResources; @@ -93,6 +95,7 @@ import org.apache.pulsar.common.policies.path.PolicyPath; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.metadata.api.coordination.LockManager; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; @@ -1199,24 +1202,43 @@ protected CompletableFuture canUpdateCluster(String tenant, Set ol /** * Redirect the call to the specified broker. * - * @param broker - * Broker name + * @param brokerId broker's id (lookup service address) */ - protected void validateBrokerName(String broker) { - String brokerUrl = String.format("http://%s", broker); - String brokerUrlTls = String.format("https://%s", broker); - if (!brokerUrl.equals(pulsar().getWebServiceAddress()) - && !brokerUrlTls.equals(pulsar().getWebServiceAddressTls())) { - String[] parts = broker.split(":"); - checkArgument(parts.length == 2, String.format("Invalid broker url %s", broker)); - String host = parts[0]; - int port = Integer.parseInt(parts[1]); - - URI redirect = UriBuilder.fromUri(uri.getRequestUri()).host(host).port(port).build(); - log.debug("[{}] Redirecting the rest call to {}: broker={}", clientAppId(), redirect, broker); - throw new WebApplicationException(Response.temporaryRedirect(redirect).build()); - + protected CompletableFuture maybeRedirectToBroker(String brokerId) { + // backwards compatibility + String cleanedBrokerId = brokerId.replaceFirst("http[s]?://", ""); + if (pulsar.getBrokerId().equals(cleanedBrokerId) + // backwards compatibility + || ("http://" + cleanedBrokerId).equals(pulsar().getWebServiceAddress()) + || ("https://" + cleanedBrokerId).equals(pulsar().getWebServiceAddressTls())) { + // no need to redirect, the current broker matches the given broker id + return CompletableFuture.completedFuture(null); } + LockManager brokerLookupDataLockManager = + pulsar().getCoordinationService().getLockManager(BrokerLookupData.class); + return brokerLookupDataLockManager.readLock(LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + cleanedBrokerId) + .thenAccept(brokerLookupDataOptional -> { + if (brokerLookupDataOptional.isEmpty()) { + throw new RestException(Status.NOT_FOUND, + "Broker id '" + brokerId + "' not found in available brokers."); + } + brokerLookupDataOptional.ifPresent(brokerLookupData -> { + URI targetBrokerUri; + if ((isRequestHttps() || StringUtils.isBlank(brokerLookupData.getWebServiceUrl())) + && StringUtils.isNotBlank(brokerLookupData.getWebServiceUrlTls())) { + targetBrokerUri = URI.create(brokerLookupData.getWebServiceUrlTls()); + } else { + targetBrokerUri = URI.create(brokerLookupData.getWebServiceUrl()); + } + URI redirect = UriBuilder.fromUri(uri.getRequestUri()) + .scheme(targetBrokerUri.getScheme()) + .host(targetBrokerUri.getHost()) + .port(targetBrokerUri.getPort()).build(); + log.debug("[{}] Redirecting the rest call to {}: broker={}", clientAppId(), redirect, + cleanedBrokerId); + throw new WebApplicationException(Response.temporaryRedirect(redirect).build()); + }); + }); } public void validateTopicPolicyOperation(TopicName topicName, PolicyName policy, PolicyOperation operation) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java index dfa65d1995381..4c24f6d303668 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java @@ -134,7 +134,11 @@ public static Optional rebatchMessage(RawMessage msg, msg.getMessageIdData().getEntryId(), msg.getMessageIdData().getPartition(), i); - if (!singleMessageMetadata.hasPartitionKey()) { + if (singleMessageMetadata.isCompactedOut()) { + // we may read compacted out message from the compacted topic + Commands.serializeSingleMessageInBatchWithPayload(emptyMetadata, + Unpooled.EMPTY_BUFFER, batchBuffer); + } else if (!singleMessageMetadata.hasPartitionKey()) { if (retainNullKey) { messagesRetained++; Commands.serializeSingleMessageInBatchWithPayload(singleMessageMetadata, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java index ba8d3db7178d9..374f1e30c0a89 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java @@ -187,6 +187,7 @@ public ByteBuf toByteBuf() { idData.writeTo(buf); buf.writeInt(metadataAndPayload.readableBytes()); buf.writeBytes(metadataAndPayload); + metadataAndPayload.release(); encryptedPayload.release(); clear(); return buf; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java index 70bda888bf7ea..f65232413991f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java @@ -59,6 +59,7 @@ public RawReaderImpl(PulsarClientImpl client, String topic, String subscription, consumerConfiguration.setReceiverQueueSize(DEFAULT_RECEIVER_QUEUE_SIZE); consumerConfiguration.setReadCompacted(true); consumerConfiguration.setSubscriptionInitialPosition(SubscriptionInitialPosition.Earliest); + consumerConfiguration.setAckReceiptEnabled(true); consumer = new RawConsumerImpl(client, consumerConfiguration, consumerFuture); @@ -122,7 +123,7 @@ static class RawConsumerImpl extends ConsumerImpl { MessageId.earliest, 0 /* startMessageRollbackDurationInSec */, Schema.BYTES, null, - true + false ); incomingRawMessages = new GrowableArrayBlockingQueue<>(); pendingRawReceives = new ConcurrentLinkedQueue<>(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java index 8794e2736d4d4..dfafbc41cb45c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java @@ -32,6 +32,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.function.Predicate; import javax.annotation.Nullable; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; @@ -100,7 +101,11 @@ public void asyncReadEntriesOrWait(ManagedCursor cursor, boolean isFirstRead, ReadEntriesCallback callback, Consumer consumer) { PositionImpl cursorPosition; - if (isFirstRead && MessageId.earliest.equals(consumer.getStartMessageId())){ + boolean readFromEarliest = isFirstRead && MessageId.earliest.equals(consumer.getStartMessageId()) + && (!cursor.isDurable() || cursor.getName().equals(Compactor.COMPACTION_SUBSCRIPTION) + || cursor.getMarkDeletedPosition() == null + || cursor.getMarkDeletedPosition().getEntryId() == -1L); + if (readFromEarliest){ cursorPosition = PositionImpl.EARLIEST; } else { cursorPosition = (PositionImpl) cursor.getReadPosition(); @@ -320,6 +325,55 @@ public CompletableFuture readLastEntryOfCompactedLedger() { }); } + CompletableFuture findFirstMatchEntry(final Predicate predicate) { + var compactedTopicContextFuture = this.getCompactedTopicContextFuture(); + + if (compactedTopicContextFuture == null) { + return CompletableFuture.completedFuture(null); + } + return compactedTopicContextFuture.thenCompose(compactedTopicContext -> { + LedgerHandle lh = compactedTopicContext.getLedger(); + CompletableFuture promise = new CompletableFuture<>(); + findFirstMatchIndexLoop(predicate, 0L, lh.getLastAddConfirmed(), promise, null, lh); + return promise.thenCompose(index -> { + if (index == null) { + return CompletableFuture.completedFuture(null); + } + return readEntries(lh, index, index).thenApply(entries -> entries.get(0)); + }); + }); + } + private static void findFirstMatchIndexLoop(final Predicate predicate, + final long start, final long end, + final CompletableFuture promise, + final Long lastMatchIndex, + final LedgerHandle lh) { + if (start > end) { + promise.complete(lastMatchIndex); + return; + } + + long mid = (start + end) / 2; + readEntries(lh, mid, mid).thenAccept(entries -> { + Entry entry = entries.get(0); + final boolean isMatch; + try { + isMatch = predicate.test(entry); + } finally { + entry.release(); + } + + if (isMatch) { + findFirstMatchIndexLoop(predicate, start, mid - 1, promise, mid, lh); + } else { + findFirstMatchIndexLoop(predicate, mid + 1, end, promise, lastMatchIndex, lh); + } + }).exceptionally(ex -> { + promise.completeExceptionally(ex); + return null; + }); + } + private static int comparePositionAndMessageId(PositionImpl p, MessageIdData m) { return ComparisonChain.start() .compare(p.getLedgerId(), m.getLedgerId()) @@ -330,6 +384,11 @@ public Optional getCompactionHorizon() { return Optional.ofNullable(this.compactionHorizon); } + public void reset() { + this.compactionHorizon = null; + this.compactedTopicContext = null; + } + @Nullable public CompletableFuture getCompactedTopicContextFuture() { return compactedTopicContext; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/PulsarTopicCompactionService.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/PulsarTopicCompactionService.java index 1d3f94dcb9048..16543bc7aa77f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/PulsarTopicCompactionService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/PulsarTopicCompactionService.java @@ -22,7 +22,6 @@ import static org.apache.pulsar.compaction.CompactedTopicImpl.COMPACT_LEDGER_EMPTY; import static org.apache.pulsar.compaction.CompactedTopicImpl.NEWER_THAN_COMPACTED; import static org.apache.pulsar.compaction.CompactedTopicImpl.findStartPoint; -import static org.apache.pulsar.compaction.CompactedTopicImpl.readEntries; import java.io.IOException; import java.util.Collections; import java.util.List; @@ -33,7 +32,6 @@ import java.util.function.Supplier; import javax.annotation.Nonnull; import org.apache.bookkeeper.client.BookKeeper; -import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; @@ -116,7 +114,7 @@ public CompletableFuture findEntryByPublishTime(long publishTime) { final Predicate predicate = entry -> { return Commands.parseMessageMetadata(entry.getDataBuffer()).getPublishTime() >= publishTime; }; - return findFirstMatchEntry(predicate); + return compactedTopic.findFirstMatchEntry(predicate); } @Override @@ -128,57 +126,7 @@ public CompletableFuture findEntryByEntryIndex(long entryIndex) { } return brokerEntryMetadata.getIndex() >= entryIndex; }; - return findFirstMatchEntry(predicate); - } - - private CompletableFuture findFirstMatchEntry(final Predicate predicate) { - var compactedTopicContextFuture = compactedTopic.getCompactedTopicContextFuture(); - - if (compactedTopicContextFuture == null) { - return CompletableFuture.completedFuture(null); - } - return compactedTopicContextFuture.thenCompose(compactedTopicContext -> { - LedgerHandle lh = compactedTopicContext.getLedger(); - CompletableFuture promise = new CompletableFuture<>(); - findFirstMatchIndexLoop(predicate, 0L, lh.getLastAddConfirmed(), promise, null, lh); - return promise.thenCompose(index -> { - if (index == null) { - return CompletableFuture.completedFuture(null); - } - return readEntries(lh, index, index).thenApply(entries -> entries.get(0)); - }); - }); - } - - private static void findFirstMatchIndexLoop(final Predicate predicate, - final long start, final long end, - final CompletableFuture promise, - final Long lastMatchIndex, - final LedgerHandle lh) { - if (start > end) { - promise.complete(lastMatchIndex); - return; - } - - long mid = (start + end) / 2; - readEntries(lh, mid, mid).thenAccept(entries -> { - Entry entry = entries.get(0); - final boolean isMatch; - try { - isMatch = predicate.test(entry); - } finally { - entry.release(); - } - - if (isMatch) { - findFirstMatchIndexLoop(predicate, start, mid - 1, promise, mid, lh); - } else { - findFirstMatchIndexLoop(predicate, mid + 1, end, promise, lastMatchIndex, lh); - } - }).exceptionally(ex -> { - promise.completeExceptionally(ex); - return null; - }); + return compactedTopic.findFirstMatchEntry(predicate); } public CompactedTopicImpl getCompactedTopic() { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SLAMonitoringTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SLAMonitoringTest.java index 47949d7312b88..4a6524bf24521 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SLAMonitoringTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SLAMonitoringTest.java @@ -102,9 +102,9 @@ void setup() throws Exception { createTenant(pulsarAdmins[BROKER_COUNT - 1]); for (int i = 0; i < BROKER_COUNT; i++) { - String topic = String.format("%s/%s/%s:%s", NamespaceService.SLA_NAMESPACE_PROPERTY, "my-cluster", - pulsarServices[i].getAdvertisedAddress(), brokerWebServicePorts[i]); - pulsarAdmins[0].namespaces().createNamespace(topic); + var namespaceName = NamespaceService.getSLAMonitorNamespace(pulsarServices[i].getBrokerId(), + pulsarServices[i].getConfig()); + pulsarAdmins[0].namespaces().createNamespace(namespaceName.toString()); } } @@ -173,9 +173,9 @@ public void testOwnedNamespaces() { public void testOwnershipViaAdminAfterSetup() { for (int i = 0; i < BROKER_COUNT; i++) { try { - String topic = String.format("persistent://%s/%s/%s:%s/%s", - NamespaceService.SLA_NAMESPACE_PROPERTY, "my-cluster", pulsarServices[i].getAdvertisedAddress(), - brokerWebServicePorts[i], "my-topic"); + String topic = String.format("persistent://%s/%s/%s/%s", + NamespaceService.SLA_NAMESPACE_PROPERTY, "my-cluster", + pulsarServices[i].getBrokerId(), "my-topic"); assertEquals(pulsarAdmins[0].lookups().lookupTopic(topic), "pulsar://" + pulsarServices[i].getAdvertisedAddress() + ":" + brokerNativeBrokerPorts[i]); } catch (Exception e) { @@ -199,8 +199,8 @@ public void testUnloadIfBrokerCrashes() { fail("Should be a able to close the broker index " + crashIndex + " Exception: " + e); } - String topic = String.format("persistent://%s/%s/%s:%s/%s", NamespaceService.SLA_NAMESPACE_PROPERTY, - "my-cluster", pulsarServices[crashIndex].getAdvertisedAddress(), brokerWebServicePorts[crashIndex], + String topic = String.format("persistent://%s/%s/%s/%s", NamespaceService.SLA_NAMESPACE_PROPERTY, + "my-cluster", pulsarServices[crashIndex].getBrokerId(), "my-topic"); log.info("Lookup for namespace {}", topic); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/TopicEventsListenerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/TopicEventsListenerTest.java index e6459bbf74c31..ceb3c1d0d9335 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/TopicEventsListenerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/TopicEventsListenerTest.java @@ -126,7 +126,7 @@ public void testEvents(String topicTypePersistence, String topicTypePartitioned, boolean forceDelete) throws Exception { String topicName = topicTypePersistence + "://" + namespace + "/" + "topic-" + UUID.randomUUID(); - createTopicAndVerifyEvents(topicTypePartitioned, topicName); + createTopicAndVerifyEvents(topicTypePersistence, topicTypePartitioned, topicName); events.clear(); if (topicTypePartitioned.equals("partitioned")) { @@ -150,7 +150,7 @@ public void testEventsWithUnload(String topicTypePersistence, String topicTypePa boolean forceDelete) throws Exception { String topicName = topicTypePersistence + "://" + namespace + "/" + "topic-" + UUID.randomUUID(); - createTopicAndVerifyEvents(topicTypePartitioned, topicName); + createTopicAndVerifyEvents(topicTypePersistence, topicTypePartitioned, topicName); events.clear(); admin.topics().unload(topicName); @@ -182,7 +182,7 @@ public void testEventsActiveSub(String topicTypePersistence, String topicTypePar boolean forceDelete) throws Exception { String topicName = topicTypePersistence + "://" + namespace + "/" + "topic-" + UUID.randomUUID(); - createTopicAndVerifyEvents(topicTypePartitioned, topicName); + createTopicAndVerifyEvents(topicTypePersistence, topicTypePartitioned, topicName); Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("sub").subscribe(); Producer producer = pulsarClient.newProducer().topic(topicName).create(); @@ -238,7 +238,7 @@ public void testEventsActiveSub(String topicTypePersistence, String topicTypePar public void testTopicAutoGC(String topicTypePersistence, String topicTypePartitioned) throws Exception { String topicName = topicTypePersistence + "://" + namespace + "/" + "topic-" + UUID.randomUUID(); - createTopicAndVerifyEvents(topicTypePartitioned, topicName); + createTopicAndVerifyEvents(topicTypePersistence, topicTypePartitioned, topicName); admin.namespaces().setInactiveTopicPolicies(namespace, new InactiveTopicPolicies(InactiveTopicDeleteMode.delete_when_no_subscriptions, 1, true)); @@ -262,25 +262,21 @@ public void testTopicAutoGC(String topicTypePersistence, String topicTypePartiti ); } - private void createTopicAndVerifyEvents(String topicTypePartitioned, String topicName) throws Exception { + private void createTopicAndVerifyEvents(String topicDomain, String topicTypePartitioned, String topicName) throws Exception { final String[] expectedEvents; - if (topicTypePartitioned.equals("partitioned")) { - topicNameToWatch = topicName + "-partition-1"; - admin.topics().createPartitionedTopic(topicName, 2); - triggerPartitionsCreation(topicName); - + if (topicDomain.equalsIgnoreCase("persistent") || topicTypePartitioned.equals("partitioned")) { expectedEvents = new String[]{ "LOAD__BEFORE", "CREATE__BEFORE", "CREATE__SUCCESS", "LOAD__SUCCESS" }; - } else { - topicNameToWatch = topicName; - admin.topics().createNonPartitionedTopic(topicName); - expectedEvents = new String[]{ + // Before https://github.com/apache/pulsar/pull/21995, Pulsar will skip create topic if the topic + // was already exists, and the action "check topic exists" will try to load Managed ledger, + // the check triggers two exrtra events: [LOAD__BEFORE, LOAD__FAILURE]. + // #21995 fixed this wrong behavior, so remove these two events. "LOAD__BEFORE", "LOAD__FAILURE", "LOAD__BEFORE", @@ -288,7 +284,14 @@ private void createTopicAndVerifyEvents(String topicTypePartitioned, String topi "CREATE__SUCCESS", "LOAD__SUCCESS" }; - + } + if (topicTypePartitioned.equals("partitioned")) { + topicNameToWatch = topicName + "-partition-1"; + admin.topics().createPartitionedTopic(topicName, 2); + triggerPartitionsCreation(topicName); + } else { + topicNameToWatch = topicName; + admin.topics().createNonPartitionedTopic(topicName); } Awaitility.waitAtMost(10, TimeUnit.SECONDS).untilAsserted(() -> diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java index 9a5d25fa0c867..3899338870451 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java @@ -516,8 +516,7 @@ public void nonPersistentTopics() throws Exception { assertEquals(topicStats.getSubscriptions().get("my-sub").getMsgDropRate(), 0); assertEquals(topicStats.getPublishers().size(), 0); assertEquals(topicStats.getMsgDropRate(), 0); - assertEquals(topicStats.getOwnerBroker(), - pulsar.getAdvertisedAddress() + ":" + pulsar.getConfiguration().getWebServicePort().get()); + assertEquals(topicStats.getOwnerBroker(), pulsar.getBrokerId()); PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(nonPersistentTopicName, false); assertEquals(internalStats.cursors.keySet(), Set.of("my-sub")); @@ -1310,7 +1309,7 @@ public void brokerNamespaceIsolationPolicies() throws Exception { String cluster = pulsar.getConfiguration().getClusterName(); String namespaceRegex = "other/" + cluster + "/other.*"; String brokerName = pulsar.getAdvertisedAddress(); - String brokerAddress = brokerName + ":" + pulsar.getConfiguration().getWebServicePort().get(); + String brokerAddress = pulsar.getBrokerId(); Map parameters1 = new HashMap<>(); parameters1.put("min_limit", "1"); @@ -1318,7 +1317,7 @@ public void brokerNamespaceIsolationPolicies() throws Exception { NamespaceIsolationData nsPolicyData1 = NamespaceIsolationData.builder() .namespaces(Collections.singletonList(namespaceRegex)) - .primary(Collections.singletonList(brokerName + ":[0-9]*")) + .primary(Collections.singletonList(brokerName)) .secondary(Collections.singletonList(brokerName + ".*")) .autoFailoverPolicy(AutoFailoverPolicyData.builder() .policyType(AutoFailoverPolicyType.min_available) @@ -3390,4 +3389,61 @@ private void testSetBacklogQuotasNamespaceLevelIfRetentionExists() throws Except // cleanup. admin.namespaces().deleteNamespace(ns); } + + @Test + private void testAnalyzeSubscriptionBacklogNotCauseStuck() throws Exception { + final String topic = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp"); + final String subscription = "s1"; + admin.topics().createNonPartitionedTopic(topic); + // Send 10 messages. + Consumer consumer = pulsarClient.newConsumer(Schema.STRING).topic(topic).subscriptionName(subscription) + .receiverQueueSize(0).subscribe(); + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + for (int i = 0; i < 10; i++) { + producer.send(i + ""); + } + + // Verify consumer can receive all messages after calling "analyzeSubscriptionBacklog". + admin.topics().analyzeSubscriptionBacklog(topic, subscription, Optional.of(MessageIdImpl.earliest)); + for (int i = 0; i < 10; i++) { + Awaitility.await().untilAsserted(() -> { + Message m = consumer.receive(); + assertNotNull(m); + consumer.acknowledge(m); + }); + } + + // cleanup. + consumer.close(); + producer.close(); + admin.topics().delete(topic); + } + + @Test + public void testGetStatsIfPartitionNotExists() throws Exception { + // create topic. + final String partitionedTp = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp"); + admin.topics().createPartitionedTopic(partitionedTp, 1); + TopicName partition0 = TopicName.get(partitionedTp).getPartition(0); + boolean topicExists1 = pulsar.getBrokerService().getTopic(partition0.toString(), false).join().isPresent(); + assertTrue(topicExists1); + // Verify topics-stats works. + TopicStats topicStats = admin.topics().getStats(partition0.toString()); + assertNotNull(topicStats); + + // Delete partition and call topic-stats again. + admin.topics().delete(partition0.toString()); + boolean topicExists2 = pulsar.getBrokerService().getTopic(partition0.toString(), false).join().isPresent(); + assertFalse(topicExists2); + // Verify: respond 404. + try { + admin.topics().getStats(partition0.toString()); + fail("Should respond 404 after the partition was deleted"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("Topic partitions were not yet created")); + } + + // cleanup. + admin.topics().deletePartitionedTopic(partitionedTp); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiHealthCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiHealthCheckTest.java index a780f889de85f..357422b11f6ce 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiHealthCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiHealthCheckTest.java @@ -23,6 +23,7 @@ import static org.testng.Assert.assertTrue; import java.lang.management.ManagementFactory; import java.lang.management.ThreadMXBean; +import java.lang.reflect.Field; import java.time.Duration; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -31,13 +32,21 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.ProducerBuilderImpl; +import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.naming.TopicVersion; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.compaction.Compactor; import org.awaitility.Awaitility; +import org.mockito.Mockito; import org.springframework.util.CollectionUtils; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -236,4 +245,58 @@ public void testHealthCheckupV2() throws Exception { )) ); } + + class DummyProducerBuilder extends ProducerBuilderImpl { + // This is a dummy producer builder to test the health check timeout + // the producer constructed by this builder will not send any message + public DummyProducerBuilder(PulsarClientImpl client, Schema schema) { + super(client, schema); + } + + @Override + public CompletableFuture> createAsync() { + CompletableFuture> future = new CompletableFuture<>(); + super.createAsync().thenAccept(producer -> { + Producer spyProducer = Mockito.spy(producer); + Mockito.doReturn(CompletableFuture.completedFuture(MessageId.earliest)) + .when(spyProducer).sendAsync(Mockito.any()); + future.complete(spyProducer); + }).exceptionally(ex -> { + future.completeExceptionally(ex); + return null; + }); + return future; + } + } + + @Test + public void testHealthCheckTimeOut() throws Exception { + final String testHealthCheckTopic = String.format("persistent://pulsar/localhost:%s/healthcheck", + pulsar.getConfig().getWebServicePort().get()); + PulsarClient client = pulsar.getClient(); + PulsarClient spyClient = Mockito.spy(client); + Mockito.doReturn(new DummyProducerBuilder<>((PulsarClientImpl) spyClient, Schema.BYTES)) + .when(spyClient).newProducer(Schema.STRING); + // use reflection to replace the client in the broker + Field field = PulsarService.class.getDeclaredField("client"); + field.setAccessible(true); + field.set(pulsar, spyClient); + try { + admin.brokers().healthcheck(TopicVersion.V2); + throw new Exception("Should not reach here"); + } catch (PulsarAdminException e) { + log.info("Exception caught", e); + assertTrue(e.getMessage().contains("LowOverheadTimeoutException")); + } + // To ensure we don't have any subscription, the producers and readers are closed. + Awaitility.await().untilAsserted(() -> + assertTrue(CollectionUtils.isEmpty(admin.topics() + .getSubscriptions(testHealthCheckTopic).stream() + // All system topics are using compaction, even though is not explicitly set in the policies. + .filter(v -> !v.equals(Compactor.COMPACTION_SUBSCRIPTION)) + .collect(Collectors.toList()) + )) + ); + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiMultiBrokersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiMultiBrokersTest.java index de4cf9658b201..7c9154a27ff69 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiMultiBrokersTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiMultiBrokersTest.java @@ -81,9 +81,8 @@ public void testGetLeaderBroker() assertTrue(leaderBroker.isPresent()); log.info("Leader broker is {}", leaderBroker); for (PulsarAdmin admin : getAllAdmins()) { - String serviceUrl = admin.brokers().getLeaderBroker().getServiceUrl(); - log.info("Pulsar admin get leader broker is {}", serviceUrl); - assertEquals(leaderBroker.get().getServiceUrl(), serviceUrl); + String brokerId = admin.brokers().getLeaderBroker().getBrokerId(); + assertEquals(leaderBroker.get().getBrokerId(), brokerId); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java index 0df378356703c..b28cfc98fdb07 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java @@ -333,10 +333,12 @@ public void clusters() throws Exception { } catch (PulsarAdminException e) { assertTrue(e instanceof PreconditionFailedException); } + + restartBroker(); } @Test - public void clusterNamespaceIsolationPolicies() throws PulsarAdminException { + public void clusterNamespaceIsolationPolicies() throws Exception { try { // create String policyName1 = "policy-1"; @@ -512,6 +514,7 @@ public void clusterNamespaceIsolationPolicies() throws PulsarAdminException { // Ok } + restartBroker(); } @Test @@ -529,7 +532,8 @@ public void brokers() throws Exception { Assert.assertEquals(list2.size(), 1); BrokerInfo leaderBroker = admin.brokers().getLeaderBroker(); - Assert.assertEquals(leaderBroker.getServiceUrl(), pulsar.getLeaderElectionService().getCurrentLeader().map(LeaderBroker::getServiceUrl).get()); + Assert.assertEquals(leaderBroker.getBrokerId(), + pulsar.getLeaderElectionService().getCurrentLeader().map(LeaderBroker::getBrokerId).get()); Map nsMap = admin.brokers().getOwnedNamespaces("test", list.get(0)); // since sla-monitor ns is not created nsMap.size() == 1 (for HeartBeat Namespace) @@ -537,7 +541,7 @@ public void brokers() throws Exception { for (String ns : nsMap.keySet()) { NamespaceOwnershipStatus nsStatus = nsMap.get(ns); if (ns.equals( - NamespaceService.getHeartbeatNamespace(pulsar.getLookupServiceAddress(), pulsar.getConfiguration()) + NamespaceService.getHeartbeatNamespace(pulsar.getBrokerId(), pulsar.getConfiguration()) + "/0x00000000_0xffffffff")) { assertEquals(nsStatus.broker_assignment, BrokerAssignment.shared); assertFalse(nsStatus.is_controlled); @@ -545,10 +549,7 @@ public void brokers() throws Exception { } } - String[] parts = list.get(0).split(":"); - Assert.assertEquals(parts.length, 2); - Map nsMap2 = adminTls.brokers().getOwnedNamespaces("test", - String.format("%s:%d", parts[0], pulsar.getListenPortHTTPS().get())); + Map nsMap2 = adminTls.brokers().getOwnedNamespaces("test", list.get(0)); Assert.assertEquals(nsMap2.size(), 2); deleteNamespaceWithRetry("prop-xyz/ns1", false); @@ -703,6 +704,10 @@ public void testInvalidDynamicConfigContentInMetadata() throws Exception { Awaitility.await().until(() -> pulsar.getConfiguration().getBrokerShutdownTimeoutMs() == newValue); // verify value is updated assertEquals(pulsar.getConfiguration().getBrokerShutdownTimeoutMs(), newValue); + // reset config + pulsar.getConfiguration().setBrokerShutdownTimeoutMs(0L); + // restart broker + restartBroker(); } /** @@ -801,6 +806,8 @@ public void namespaces() throws Exception { TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test", "usw")); admin.tenants().updateTenant("prop-xyz", tenantInfo); + Awaitility.await().untilAsserted(() -> + assertEquals(admin.tenants().getTenantInfo("prop-xyz").getAllowedClusters(), Set.of("test", "usw"))); assertEquals(admin.namespaces().getPolicies("prop-xyz/ns1").bundles, PoliciesUtil.defaultBundle()); @@ -933,8 +940,7 @@ public void persistentTopics(String topicName) throws Exception { assertEquals(topicStats.getSubscriptions().get(subName).getConsumers().size(), 1); assertEquals(topicStats.getSubscriptions().get(subName).getMsgBacklog(), 10); assertEquals(topicStats.getPublishers().size(), 0); - assertEquals(topicStats.getOwnerBroker(), - pulsar.getAdvertisedAddress() + ":" + pulsar.getConfiguration().getWebServicePortTls().get()); + assertEquals(topicStats.getOwnerBroker(), pulsar.getBrokerId()); PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(persistentTopicName, false); assertEquals(internalStats.cursors.keySet(), Set.of(Codec.encode(subName))); @@ -3191,6 +3197,9 @@ public void testTopicBundleRangeLookup() throws PulsarAdminException, PulsarServ TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test", "usw")); admin.tenants().updateTenant("prop-xyz", tenantInfo); + Awaitility.await().untilAsserted(() -> + assertEquals(admin.tenants().getTenantInfo("prop-xyz").getAllowedClusters(), + tenantInfo.getAllowedClusters())); admin.namespaces().createNamespace("prop-xyz/getBundleNs", 100); assertEquals(admin.namespaces().getPolicies("prop-xyz/getBundleNs").bundles.getNumBundles(), 100); @@ -3384,6 +3393,9 @@ public void testCreateAndDeleteNamespaceWithBundles() throws Exception { TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test", "usw")); admin.tenants().updateTenant("prop-xyz", tenantInfo); + Awaitility.await().untilAsserted(() -> + assertEquals(admin.tenants().getTenantInfo("prop-xyz").getAllowedClusters(), + tenantInfo.getAllowedClusters())); String ns = BrokerTestUtil.newUniqueName("prop-xyz/ns"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java index 3caeb591bc8f3..2894903c0d0c1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java @@ -302,7 +302,7 @@ public void clusters() throws Exception { NamespaceIsolationDataImpl policyData = NamespaceIsolationDataImpl.builder() .namespaces(Collections.singletonList("dummy/colo/ns")) - .primary(Collections.singletonList("localhost" + ":" + pulsar.getListenPortHTTP())) + .primary(Collections.singletonList(pulsar.getAdvertisedAddress())) .autoFailoverPolicy(AutoFailoverPolicyData.builder() .policyType(AutoFailoverPolicyType.min_available) .parameters(parameters1) @@ -722,11 +722,12 @@ public void brokers() throws Exception { assertTrue(res instanceof Set); Set activeBrokers = (Set) res; assertEquals(activeBrokers.size(), 1); - assertEquals(activeBrokers, Set.of(pulsar.getAdvertisedAddress() + ":" + pulsar.getListenPortHTTP().get())); + assertEquals(activeBrokers, Set.of(pulsar.getBrokerId())); Object leaderBrokerRes = asyncRequests(ctx -> brokers.getLeaderBroker(ctx)); assertTrue(leaderBrokerRes instanceof BrokerInfo); BrokerInfo leaderBroker = (BrokerInfo)leaderBrokerRes; - assertEquals(leaderBroker.getServiceUrl(), pulsar.getLeaderElectionService().getCurrentLeader().map(LeaderBroker::getServiceUrl).get()); + assertEquals(leaderBroker.getBrokerId(), + pulsar.getLeaderElectionService().getCurrentLeader().map(LeaderBroker::getBrokerId).get()); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AnalyzeBacklogSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AnalyzeBacklogSubscriptionTest.java index 64b2a58ab86e8..f8aa3dc355d92 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AnalyzeBacklogSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AnalyzeBacklogSubscriptionTest.java @@ -154,17 +154,17 @@ private void verifyBacklog(String topic, String subscription, int numEntries, in AnalyzeSubscriptionBacklogResult analyzeSubscriptionBacklogResult = admin.topics().analyzeSubscriptionBacklog(topic, subscription, Optional.empty()); - assertEquals(numEntries, analyzeSubscriptionBacklogResult.getEntries()); - assertEquals(numEntries, analyzeSubscriptionBacklogResult.getFilterAcceptedEntries()); - assertEquals(0, analyzeSubscriptionBacklogResult.getFilterRejectedEntries()); - assertEquals(0, analyzeSubscriptionBacklogResult.getFilterRescheduledEntries()); - assertEquals(0, analyzeSubscriptionBacklogResult.getFilterRescheduledEntries()); + assertEquals(analyzeSubscriptionBacklogResult.getEntries(), numEntries); + assertEquals(analyzeSubscriptionBacklogResult.getFilterAcceptedEntries(), numEntries); + assertEquals(analyzeSubscriptionBacklogResult.getFilterRejectedEntries(), 0); + assertEquals(analyzeSubscriptionBacklogResult.getFilterRescheduledEntries(), 0); + assertEquals(analyzeSubscriptionBacklogResult.getFilterRescheduledEntries(), 0); - assertEquals(numMessages, analyzeSubscriptionBacklogResult.getMessages()); - assertEquals(numMessages, analyzeSubscriptionBacklogResult.getFilterAcceptedMessages()); - assertEquals(0, analyzeSubscriptionBacklogResult.getFilterRejectedMessages()); + assertEquals(analyzeSubscriptionBacklogResult.getMessages(), numMessages); + assertEquals(analyzeSubscriptionBacklogResult.getFilterAcceptedMessages(), numMessages); + assertEquals(analyzeSubscriptionBacklogResult.getFilterRejectedMessages(), 0); - assertEquals(0, analyzeSubscriptionBacklogResult.getFilterRescheduledMessages()); + assertEquals(analyzeSubscriptionBacklogResult.getFilterRescheduledMessages(), 0); assertFalse(analyzeSubscriptionBacklogResult.isAborted()); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 7939b19283946..23cb413614f9a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -31,6 +31,8 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertSame; import static org.testng.Assert.assertTrue; import java.lang.reflect.Field; import java.util.ArrayList; @@ -65,6 +67,7 @@ import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.web.PulsarWebResource; import org.apache.pulsar.broker.web.RestException; +import org.apache.pulsar.client.admin.LongRunningProcessStatus; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.admin.Topics; @@ -87,6 +90,7 @@ import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.AuthAction; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.TenantInfoImpl; @@ -1364,21 +1368,12 @@ public void testGetMessageById() throws Exception { Message message2 = admin.topics().getMessageById(topicName2, id2.getLedgerId(), id2.getEntryId()); Assert.assertEquals(message2.getData(), data2.getBytes()); - Message message3 = null; - try { - message3 = admin.topics().getMessageById(topicName2, id1.getLedgerId(), id1.getEntryId()); - Assert.fail(); - } catch (Exception e) { - Assert.assertNull(message3); - } - - Message message4 = null; - try { - message4 = admin.topics().getMessageById(topicName1, id2.getLedgerId(), id2.getEntryId()); - Assert.fail(); - } catch (Exception e) { - Assert.assertNull(message4); - } + Assert.expectThrows(PulsarAdminException.NotFoundException.class, () -> { + admin.topics().getMessageById(topicName2, id1.getLedgerId(), id1.getEntryId()); + }); + Assert.expectThrows(PulsarAdminException.NotFoundException.class, () -> { + admin.topics().getMessageById(topicName1, id2.getLedgerId(), id2.getEntryId()); + }); } @Test @@ -1459,6 +1454,69 @@ public void onSendAcknowledgement(Producer producer, Message message, MessageId .compareTo(id2) > 0); } + @Test + public void testGetMessageIdByTimestampWithCompaction() throws Exception { + TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test")); + admin.tenants().createTenant("tenant-xyz", tenantInfo); + admin.namespaces().createNamespace("tenant-xyz/ns-abc", Set.of("test")); + final String topicName = "persistent://tenant-xyz/ns-abc/testGetMessageIdByTimestampWithCompaction"; + admin.topics().createNonPartitionedTopic(topicName); + + Map publishTimeMap = new ConcurrentHashMap<>(); + @Cleanup + ProducerBase producer = (ProducerBase) pulsarClient.newProducer().topic(topicName) + .enableBatching(false) + .intercept(new ProducerInterceptor() { + @Override + public void close() { + + } + + @Override + public boolean eligible(Message message) { + return true; + } + + @Override + public Message beforeSend(Producer producer, Message message) { + return message; + } + + @Override + public void onSendAcknowledgement(Producer producer, Message message, MessageId msgId, + Throwable exception) { + publishTimeMap.put(message.getMessageId(), message.getPublishTime()); + } + }) + .create(); + + MessageId id1 = producer.newMessage().key("K1").value("test1".getBytes()).send(); + MessageId id2 = producer.newMessage().key("K2").value("test2".getBytes()).send(); + + long publish1 = publishTimeMap.get(id1); + long publish2 = publishTimeMap.get(id2); + Assert.assertTrue(publish1 < publish2); + + admin.topics().triggerCompaction(topicName); + Awaitility.await().untilAsserted(() -> + assertSame(admin.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS)); + + admin.topics().unload(topicName); + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(topicName, false); + assertEquals(internalStats.ledgers.size(), 1); + assertEquals(internalStats.ledgers.get(0).entries, 0); + }); + + Assert.assertEquals(admin.topics().getMessageIdByTimestamp(topicName, publish1 - 1), id1); + Assert.assertEquals(admin.topics().getMessageIdByTimestamp(topicName, publish1), id1); + Assert.assertEquals(admin.topics().getMessageIdByTimestamp(topicName, publish1 + 1), id2); + Assert.assertEquals(admin.topics().getMessageIdByTimestamp(topicName, publish2), id2); + Assert.assertTrue(admin.topics().getMessageIdByTimestamp(topicName, publish2 + 1) + .compareTo(id2) > 0); + } + @Test public void testGetBatchMessageIdByTimestamp() throws Exception { TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test")); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAutoCreationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAutoCreationTest.java index 9cd1cf214f67e..bb4a23bf24bd9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAutoCreationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAutoCreationTest.java @@ -149,10 +149,11 @@ public void testPartitionedTopicAutoCreationForbiddenDuringNamespaceDeletion() .sendTimeout(1, TimeUnit.SECONDS) .topic(topic) .create()) { - } catch (PulsarClientException.LookupException expected) { - String msg = "Namespace bundle for topic (%s) not served by this instance"; + } catch (PulsarClientException.TopicDoesNotExistException expected) { + // Since the "policies.deleted" is "true", the value of "isAllowAutoTopicCreationAsync" will be false, + // so the "TopicDoesNotExistException" is expected. log.info("Expected error", expected); - assertTrue(expected.getMessage().contains(String.format(msg, topic)) + assertTrue(expected.getMessage().contains(topic) || expected.getMessage().contains(topicPoliciesServiceInitException)); } @@ -160,10 +161,11 @@ public void testPartitionedTopicAutoCreationForbiddenDuringNamespaceDeletion() .topic(topic) .subscriptionName("test") .subscribe()) { - } catch (PulsarClientException.LookupException expected) { - String msg = "Namespace bundle for topic (%s) not served by this instance"; + } catch (PulsarClientException.TopicDoesNotExistException expected) { + // Since the "policies.deleted" is "true", the value of "isAllowAutoTopicCreationAsync" will be false, + // so the "TopicDoesNotExistException" is expected. log.info("Expected error", expected); - assertTrue(expected.getMessage().contains(String.format(msg, topic)) + assertTrue(expected.getMessage().contains(topic) || expected.getMessage().contains(topicPoliciesServiceInitException)); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesAuthZTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesAuthZTest.java new file mode 100644 index 0000000000000..f07b9a6c2aabf --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesAuthZTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin; + +import io.jsonwebtoken.Jwts; +import java.util.Set; +import java.util.UUID; +import lombok.Cleanup; +import lombok.SneakyThrows; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; +import org.apache.pulsar.common.policies.data.AuthAction; +import org.apache.pulsar.common.policies.data.RetentionPolicies; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.security.MockedPulsarStandalone; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static org.awaitility.Awaitility.await; + + +public final class TopicPoliciesAuthZTest extends MockedPulsarStandalone { + + private PulsarAdmin superUserAdmin; + + private PulsarAdmin tenantManagerAdmin; + + private static final String TENANT_ADMIN_SUBJECT = UUID.randomUUID().toString(); + private static final String TENANT_ADMIN_TOKEN = Jwts.builder() + .claim("sub", TENANT_ADMIN_SUBJECT).signWith(SECRET_KEY).compact(); + + @SneakyThrows + @BeforeClass + public void before() { + configureTokenAuthentication(); + configureDefaultAuthorization(); + start(); + this.superUserAdmin =PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(SUPER_USER_TOKEN)) + .build(); + final TenantInfo tenantInfo = superUserAdmin.tenants().getTenantInfo("public"); + tenantInfo.getAdminRoles().add(TENANT_ADMIN_SUBJECT); + superUserAdmin.tenants().updateTenant("public", tenantInfo); + this.tenantManagerAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(TENANT_ADMIN_TOKEN)) + .build(); + } + + + @SneakyThrows + @AfterClass + public void after() { + close(); + } + + + @SneakyThrows + @Test + public void testRetention() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createNonPartitionedTopic(topic); + + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + final RetentionPolicies definedRetentionPolicy = new RetentionPolicies(1, 1); + // test superuser + superUserAdmin.topicPolicies().setRetention(topic, definedRetentionPolicy); + + // because the topic policies is eventual consistency, we should wait here + await().untilAsserted(() -> { + final RetentionPolicies receivedRetentionPolicy = superUserAdmin.topicPolicies().getRetention(topic); + Assert.assertEquals(receivedRetentionPolicy, definedRetentionPolicy); + }); + superUserAdmin.topicPolicies().removeRetention(topic); + + await().untilAsserted(() -> { + final RetentionPolicies retention = superUserAdmin.topicPolicies().getRetention(topic); + Assert.assertNull(retention); + }); + + // test tenant manager + + tenantManagerAdmin.topicPolicies().setRetention(topic, definedRetentionPolicy); + await().untilAsserted(() -> { + final RetentionPolicies receivedRetentionPolicy = tenantManagerAdmin.topicPolicies().getRetention(topic); + Assert.assertEquals(receivedRetentionPolicy, definedRetentionPolicy); + }); + tenantManagerAdmin.topicPolicies().removeRetention(topic); + await().untilAsserted(() -> { + final RetentionPolicies retention = tenantManagerAdmin.topicPolicies().getRetention(topic); + Assert.assertNull(retention); + }); + + // test nobody + + try { + subAdmin.topicPolicies().getRetention(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + + subAdmin.topicPolicies().setRetention(topic, definedRetentionPolicy); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + subAdmin.topicPolicies().removeRetention(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + // test sub user with permissions + for (AuthAction action : AuthAction.values()) { + superUserAdmin.namespaces().grantPermissionOnNamespace("public/default", + subject, Set.of(action)); + try { + subAdmin.topicPolicies().getRetention(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + + subAdmin.topicPolicies().setRetention(topic, definedRetentionPolicy); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + subAdmin.topicPolicies().removeRetention(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + superUserAdmin.namespaces().revokePermissionsOnNamespace("public/default", subject); + } + } + +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest.java index d9f2d41a30b6b..f2faa98636ba2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest.java @@ -452,7 +452,7 @@ public void brokers() throws Exception { for (String ns : nsMap.keySet()) { NamespaceOwnershipStatus nsStatus = nsMap.get(ns); if (ns.equals( - NamespaceService.getHeartbeatNamespace(pulsar.getLookupServiceAddress(), pulsar.getConfiguration()) + NamespaceService.getHeartbeatNamespace(pulsar.getBrokerId(), pulsar.getConfiguration()) + "/0x00000000_0xffffffff")) { assertEquals(nsStatus.broker_assignment, BrokerAssignment.shared); assertFalse(nsStatus.is_controlled); @@ -460,10 +460,7 @@ public void brokers() throws Exception { } } - String[] parts = list.get(0).split(":"); - Assert.assertEquals(parts.length, 2); - Map nsMap2 = adminTls.brokers().getOwnedNamespaces("use", - String.format("%s:%d", parts[0], pulsar.getListenPortHTTPS().get())); + Map nsMap2 = adminTls.brokers().getOwnedNamespaces("use", list.get(0)); Assert.assertEquals(nsMap2.size(), 2); admin.namespaces().deleteNamespace("prop-xyz/use/ns1"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java index b8d75bd0fbcac..bd08ced1e0366 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java @@ -22,6 +22,7 @@ import static org.testng.Assert.assertEquals; import com.google.common.collect.Sets; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.net.InetSocketAddress; import java.net.URI; import java.net.URL; @@ -55,7 +56,9 @@ import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.ProducerImpl; +import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.auth.AuthenticationDisabled; import org.apache.pulsar.client.impl.auth.AuthenticationTls; import org.apache.pulsar.common.policies.data.ClusterData; @@ -708,5 +711,25 @@ public static class ServiceProducer { private PersistentTopic persistentTopic; } + protected void sleepSeconds(int seconds){ + try { + Thread.sleep(1000 * seconds); + } catch (InterruptedException e) { + log.warn("This thread has been interrupted", e); + Thread.currentThread().interrupt(); + } + } + + private static void reconnectAllConnections(PulsarClientImpl c) throws Exception { + ConnectionPool pool = c.getCnxPool(); + Method closeAllConnections = ConnectionPool.class.getDeclaredMethod("closeAllConnections", new Class[]{}); + closeAllConnections.setAccessible(true); + closeAllConnections.invoke(pool, new Object[]{}); + } + + protected void reconnectAllConnections() throws Exception { + reconnectAllConnections((PulsarClientImpl) pulsarClient); + } + private static final Logger log = LoggerFactory.getLogger(MockedPulsarServiceBaseTest.class); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AdvertisedListenersMultiBrokerLeaderElectionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AdvertisedListenersMultiBrokerLeaderElectionTest.java new file mode 100644 index 0000000000000..5adc78b2c5212 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AdvertisedListenersMultiBrokerLeaderElectionTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance; + +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class AdvertisedListenersMultiBrokerLeaderElectionTest extends MultiBrokerLeaderElectionTest { + @Override + protected PulsarTestContext.Builder createPulsarTestContextBuilder(ServiceConfiguration conf) { + conf.setWebServicePortTls(Optional.of(0)); + return super.createPulsarTestContextBuilder(conf).preallocatePorts(true).configOverride(config -> { + // use advertised address that is different than the name used in the advertised listeners + config.setAdvertisedAddress("localhost"); + config.setAdvertisedListeners( + "public_pulsar:pulsar://127.0.0.1:" + config.getBrokerServicePort().get() + + ",public_http:http://127.0.0.1:" + config.getWebServicePort().get() + + ",public_https:https://127.0.0.1:" + config.getWebServicePortTls().get()); + }); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java index 560cfa9216a02..5fbda961c0e3d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java @@ -103,14 +103,14 @@ public void setup() throws Exception { setupConfigs(conf); super.internalSetup(conf); pulsar1 = pulsar; - primaryHost = String.format("%s:%d", "localhost", pulsar1.getListenPortHTTP().get()); + primaryHost = pulsar1.getBrokerId(); admin1 = admin; var config2 = getDefaultConf(); setupConfigs(config2); additionalPulsarTestContext = createAdditionalPulsarTestContext(config2); pulsar2 = additionalPulsarTestContext.getPulsarService(); - secondaryHost = String.format("%s:%d", "localhost", pulsar2.getListenPortHTTP().get()); + secondaryHost = pulsar2.getBrokerId(); primaryLoadManager = getField(pulsar1.getLoadManager().get(), "loadManager"); secondaryLoadManager = getField(pulsar2.getLoadManager().get(), "loadManager"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java index 008897136f8cf..ded4ee8e58d53 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java @@ -115,7 +115,8 @@ public void anErrorShouldBeThrowBeforeLeaderElected() throws PulsarServerExcepti checkLookupException(tenant, namespace, client); // broker, webService and leaderElectionService is started, and elect is done; - leaderBrokerReference.set(new LeaderBroker(pulsar.getWebServiceAddress())); + leaderBrokerReference.set( + new LeaderBroker(pulsar.getBrokerId(), pulsar.getSafeWebServiceAddress())); Producer producer = client.newProducer() .topic("persistent://" + tenant + "/" + namespace + "/1p") diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadBalancerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadBalancerTest.java index 50afb71ea0967..7a2314b01a3d1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadBalancerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadBalancerTest.java @@ -133,7 +133,7 @@ void setup() throws Exception { brokerNativeBrokerPorts[i] = pulsarServices[i].getBrokerListenPort().get(); brokerUrls[i] = new URL("http://127.0.0.1" + ":" + brokerWebServicePorts[i]); - lookupAddresses[i] = pulsarServices[i].getAdvertisedAddress() + ":" + pulsarServices[i].getListenPortHTTP().get(); + lookupAddresses[i] = pulsarServices[i].getBrokerId(); pulsarAdmins[i] = PulsarAdmin.builder().serviceHttpUrl(brokerUrls[i].toString()).build(); } @@ -401,7 +401,7 @@ public void testTopicAssignmentWithExistingBundles() throws Exception { double expectedMaxVariation = 10.0; for (int i = 0; i < BROKER_COUNT; i++) { long actualValue = 0; - String resourceId = "http://" + lookupAddresses[i]; + String resourceId = lookupAddresses[i]; if (namespaceOwner.containsKey(resourceId)) { actualValue = namespaceOwner.get(resourceId); } @@ -681,7 +681,7 @@ public void testLeaderElection() throws Exception { } } // Make sure all brokers see the same leader - log.info("Old leader is : {}", oldLeader.getServiceUrl()); + log.info("Old leader is : {}", oldLeader.getBrokerId()); for (PulsarService pulsar : activePulsar) { log.info("Current leader for {} is : {}", pulsar.getWebServiceAddress(), pulsar.getLeaderElectionService().getCurrentLeader()); assertEquals(pulsar.getLeaderElectionService().readCurrentLeader().join(), Optional.of(oldLeader)); @@ -691,7 +691,7 @@ public void testLeaderElection() throws Exception { leaderPulsar.close(); loopUntilLeaderChangesForAllBroker(followerPulsar, oldLeader); LeaderBroker newLeader = followerPulsar.get(0).getLeaderElectionService().readCurrentLeader().join().get(); - log.info("New leader is : {}", newLeader.getServiceUrl()); + log.info("New leader is : {}", newLeader.getBrokerId()); Assert.assertNotEquals(newLeader, oldLeader); } } @@ -729,7 +729,7 @@ private void createNamespacePolicies(PulsarService pulsar) throws Exception { // set up policy that use this broker as secondary policyData = NamespaceIsolationData.builder() .namespaces(Collections.singletonList("pulsar/use/secondary-ns.*")) - .primary(Collections.singletonList(pulsarServices[0].getWebServiceAddress())) + .primary(Collections.singletonList(pulsarServices[0].getAdvertisedAddress())) .secondary(allExceptFirstBroker) .autoFailoverPolicy(AutoFailoverPolicyData.builder() .policyType(AutoFailoverPolicyType.min_available) @@ -741,7 +741,7 @@ private void createNamespacePolicies(PulsarService pulsar) throws Exception { // set up policy that do not use this broker (neither primary nor secondary) policyData = NamespaceIsolationData.builder() .namespaces(Collections.singletonList("pulsar/use/shared-ns.*")) - .primary(Collections.singletonList(pulsarServices[0].getWebServiceAddress())) + .primary(Collections.singletonList(pulsarServices[0].getAdvertisedAddress())) .secondary(allExceptFirstBroker) .autoFailoverPolicy(AutoFailoverPolicyData.builder() .policyType(AutoFailoverPolicyType.min_available) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionTest.java index 5c840129dd8d5..a7eaffc147b3f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionTest.java @@ -20,6 +20,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -36,14 +37,24 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.MultiBrokerTestZKBaseTest; import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.client.admin.Lookup; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.impl.LookupService; +import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.common.naming.TopicName; import org.awaitility.Awaitility; import org.testng.annotations.Test; @Slf4j @Test(groups = "broker") public class MultiBrokerLeaderElectionTest extends MultiBrokerTestZKBaseTest { + public MultiBrokerLeaderElectionTest() { + super(); + this.isTcpLookup = true; + } + @Override protected int numberOfAdditionalBrokers() { return 9; @@ -88,39 +99,80 @@ public void shouldAllBrokersBeAbleToGetTheLeader() { }); } - @Test - public void shouldProvideConsistentAnswerToTopicLookups() + @Test(timeOut = 60000L) + public void shouldProvideConsistentAnswerToTopicLookupsUsingAdminApi() throws PulsarAdminException, ExecutionException, InterruptedException { - String topicNameBase = "persistent://public/default/lookuptest" + UUID.randomUUID() + "-"; + String namespace = "public/ns" + UUID.randomUUID(); + admin.namespaces().createNamespace(namespace, 256); + String topicNameBase = "persistent://" + namespace + "/lookuptest-"; List topicNames = IntStream.range(0, 500).mapToObj(i -> topicNameBase + i) .collect(Collectors.toList()); List allAdmins = getAllAdmins(); @Cleanup("shutdown") ExecutorService executorService = Executors.newFixedThreadPool(allAdmins.size()); List>> resultFutures = new ArrayList<>(); - String leaderBrokerUrl = admin.brokers().getLeaderBroker().getServiceUrl(); - log.info("LEADER is {}", leaderBrokerUrl); // use Phaser to increase the chances of a race condition by triggering all threads once - // they are waiting just before the lookupTopic call + // they are waiting just before each lookupTopic call final Phaser phaser = new Phaser(1); for (PulsarAdmin brokerAdmin : allAdmins) { - if (!leaderBrokerUrl.equals(brokerAdmin.getServiceUrl())) { - phaser.register(); - log.info("Doing lookup to broker {}", brokerAdmin.getServiceUrl()); - resultFutures.add(executorService.submit(() -> { - phaser.arriveAndAwaitAdvance(); - return topicNames.stream().map(topicName -> { - try { - return brokerAdmin.lookups().lookupTopic(topicName); - } catch (PulsarAdminException e) { - log.error("Error looking up topic {} in {}", topicName, brokerAdmin.getServiceUrl()); - throw new RuntimeException(e); - } - }).collect(Collectors.toList()); - })); + phaser.register(); + Lookup lookups = brokerAdmin.lookups(); + log.info("Doing lookup to broker {}", brokerAdmin.getServiceUrl()); + resultFutures.add(executorService.submit(() -> topicNames.stream().map(topicName -> { + phaser.arriveAndAwaitAdvance(); + try { + return lookups.lookupTopic(topicName); + } catch (PulsarAdminException e) { + log.error("Error looking up topic {} in {}", topicName, brokerAdmin.getServiceUrl()); + throw new RuntimeException(e); + } + }).collect(Collectors.toList()))); + } + phaser.arriveAndDeregister(); + List firstResult = null; + for (Future> resultFuture : resultFutures) { + List result = resultFuture.get(); + if (firstResult == null) { + firstResult = result; + } else { + assertEquals(result, firstResult, "The lookup results weren't consistent."); } } - phaser.arriveAndAwaitAdvance(); + } + + @Test(timeOut = 60000L) + public void shouldProvideConsistentAnswerToTopicLookupsUsingClient() + throws PulsarAdminException, ExecutionException, InterruptedException { + String namespace = "public/ns" + UUID.randomUUID(); + admin.namespaces().createNamespace(namespace, 256); + String topicNameBase = "persistent://" + namespace + "/lookuptest-"; + List topicNames = IntStream.range(0, 500).mapToObj(i -> topicNameBase + i) + .collect(Collectors.toList()); + List allClients = getAllClients(); + @Cleanup("shutdown") + ExecutorService executorService = Executors.newFixedThreadPool(allClients.size()); + List>> resultFutures = new ArrayList<>(); + // use Phaser to increase the chances of a race condition by triggering all threads once + // they are waiting just before each lookupTopic call + final Phaser phaser = new Phaser(1); + for (PulsarClient brokerClient : allClients) { + phaser.register(); + String serviceUrl = ((PulsarClientImpl) brokerClient).getConfiguration().getServiceUrl(); + LookupService lookupService = ((PulsarClientImpl) brokerClient).getLookup(); + log.info("Doing lookup to broker {}", serviceUrl); + resultFutures.add(executorService.submit(() -> topicNames.stream().map(topicName -> { + phaser.arriveAndAwaitAdvance(); + try { + InetSocketAddress logicalAddress = + lookupService.getBroker(TopicName.get(topicName)).get().getLogicalAddress(); + return logicalAddress.getHostString() + ":" + logicalAddress.getPort(); + } catch (InterruptedException | ExecutionException e) { + log.error("Error looking up topic {} in {}", topicName, serviceUrl); + throw new RuntimeException(e); + } + }).collect(Collectors.toList()))); + } + phaser.arriveAndDeregister(); List firstResult = null; for (Future> resultFuture : resultFutures) { List result = resultFuture.get(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java index 43706129fbe15..f6154e3ec8e30 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java @@ -24,7 +24,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import com.fasterxml.jackson.databind.ObjectMapper; @@ -213,7 +213,7 @@ public void testBasicBrokerSelection() throws Exception { rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024)); rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024)); - ResourceUnit ru1 = new SimpleResourceUnit("http://prod2-broker7.messaging.usw.example.com:8080", rd); + ResourceUnit ru1 = new SimpleResourceUnit("prod2-broker7.messaging.usw.example.com:8080", rd); Set rus = new HashSet<>(); rus.add(ru1); LoadRanker lr = new ResourceAvailabilityRanker(); @@ -249,15 +249,15 @@ public void testPrimary() throws Exception { rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024)); rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024)); - ResourceUnit ru1 = new SimpleResourceUnit( - "http://" + pulsar1.getAdvertisedAddress() + ":" + pulsar1.getConfiguration().getWebServicePort().get(), rd); + ResourceUnit ru1 = new SimpleResourceUnit(pulsar1.getBrokerId(), rd); Set rus = new HashSet<>(); rus.add(ru1); LoadRanker lr = new ResourceAvailabilityRanker(); // inject the load report and rankings Map loadReports = new HashMap<>(); - org.apache.pulsar.policies.data.loadbalancer.LoadReport loadReport = new org.apache.pulsar.policies.data.loadbalancer.LoadReport(); + org.apache.pulsar.policies.data.loadbalancer.LoadReport loadReport = + new org.apache.pulsar.policies.data.loadbalancer.LoadReport(); loadReport.setSystemResourceUsage(new SystemResourceUsage()); loadReports.put(ru1, loadReport); setObjectField(SimpleLoadManagerImpl.class, loadManager, "currentLoadReports", loadReports); @@ -272,10 +272,9 @@ public void testPrimary() throws Exception { sortedRankingsInstance.get().put(lr.getRank(rd), rus); setObjectField(SimpleLoadManagerImpl.class, loadManager, "sortedRankings", sortedRankingsInstance); - final Optional leastLoaded = loadManager.getLeastLoaded(NamespaceName.get("pulsar/use/primary-ns.10")); - // broker is not active so found should be null - assertFalse(leastLoaded.isPresent()); - + ResourceUnit found = loadManager.getLeastLoaded(NamespaceName.get("pulsar/use/primary-ns.10")).get(); + // TODO: this test doesn't make sense. This was the original assertion. + assertNotEquals(found, null, "did not find a broker when expected one to be found"); } @Test(enabled = false) @@ -290,7 +289,7 @@ public void testPrimarySecondary() throws Exception { rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024)); rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024)); - ResourceUnit ru1 = new SimpleResourceUnit("http://prod2-broker7.messaging.usw.example.com:8080", rd); + ResourceUnit ru1 = new SimpleResourceUnit("prod2-broker7.messaging.usw.example.com:8080", rd); Set rus = new HashSet<>(); rus.add(ru1); LoadRanker lr = new ResourceAvailabilityRanker(); @@ -360,8 +359,8 @@ public void testDoLoadShedding() throws Exception { rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024)); rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024)); - ResourceUnit ru1 = new SimpleResourceUnit("http://pulsar-broker1.com:8080", rd); - ResourceUnit ru2 = new SimpleResourceUnit("http://pulsar-broker2.com:8080", rd); + ResourceUnit ru1 = new SimpleResourceUnit("pulsar-broker1.com:8080", rd); + ResourceUnit ru2 = new SimpleResourceUnit("pulsar-broker2.com:8080", rd); Set rus = new HashSet<>(); rus.add(ru1); rus.add(ru2); @@ -414,22 +413,18 @@ public void testEvenBundleDistribution() throws Exception { final SimpleLoadManagerImpl loadManager = (SimpleLoadManagerImpl) pulsar1.getLoadManager().get(); for (final NamespaceBundle bundle : bundles) { - if (loadManager.getLeastLoaded(bundle).get().getResourceId().equals(getAddress(primaryTlsHost))) { + if (loadManager.getLeastLoaded(bundle).get().getResourceId().equals(pulsar1.getBrokerId())) { ++numAssignedToPrimary; } else { ++numAssignedToSecondary; } // Check that number of assigned bundles are equivalent when an even number have been assigned. if ((numAssignedToPrimary + numAssignedToSecondary) % 2 == 0) { - assert (numAssignedToPrimary == numAssignedToSecondary); + assertEquals(numAssignedToPrimary, numAssignedToSecondary); } } } - private static String getAddress(String url) { - return url.replaceAll("https", "http"); - } - @Test public void testNamespaceBundleStats() { NamespaceBundleStats nsb1 = new NamespaceBundleStats(); @@ -519,7 +514,8 @@ public void testRedirectOwner() throws PulsarAdminException { } private void setupClusters() throws PulsarAdminException { - admin1.clusters().createCluster("use", ClusterData.builder().serviceUrl(pulsar1.getWebServiceAddress()).build()); + admin1.clusters().createCluster("use", ClusterData.builder().serviceUrl(pulsar1.getWebServiceAddress()) + .brokerServiceUrl(pulsar1.getBrokerServiceUrl()).build()); TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("use")); defaultTenant = "prop-xyz"; admin1.tenants().createTenant(defaultTenant, tenantInfo); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java index fca41837b9df4..fdd1eb7272c30 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java @@ -75,7 +75,7 @@ public class BrokerRegistryTest { private LocalBookkeeperEnsemble bkEnsemble; - // Make sure the load manager don't register itself to `/loadbalance/brokers/{lookupServiceAddress}` + // Make sure the load manager don't register itself to `/loadbalance/brokers/{brokerId}`. public static class MockLoadManager implements LoadManager { @Override @@ -291,7 +291,7 @@ public void testRegisterFailWithSameBrokerId() throws Exception { pulsar1.start(); pulsar2.start(); - doReturn(pulsar1.getLookupServiceAddress()).when(pulsar2).getLookupServiceAddress(); + doReturn(pulsar1.getBrokerId()).when(pulsar2).getBrokerId(); BrokerRegistryImpl brokerRegistry1 = createBrokerRegistryImpl(pulsar1); BrokerRegistryImpl brokerRegistry2 = createBrokerRegistryImpl(pulsar2); brokerRegistry1.start(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index bb7416ddc4103..b6073e434c44a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.broker.loadbalance.extensions; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Releasing; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelTest.overrideTableView; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Admin; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Bandwidth; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.MsgRate; @@ -87,6 +89,7 @@ import org.apache.pulsar.broker.loadbalance.LeaderElectionService; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; @@ -105,6 +108,7 @@ import org.apache.pulsar.broker.namespace.NamespaceBundleSplitListener; import org.apache.pulsar.broker.namespace.NamespaceEphemeralData; import org.apache.pulsar.broker.namespace.NamespaceService; +import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; @@ -134,6 +138,7 @@ import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; import org.awaitility.Awaitility; +import org.testng.AssertJUnit; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; @@ -164,13 +169,7 @@ public class ExtensibleLoadManagerImplTest extends MockedPulsarServiceBaseTest { private LookupService lookupService; - @BeforeClass - @Override - public void setup() throws Exception { - // Set the inflight state waiting time and ownership monitor delay time to 5 seconds to avoid - // stuck when doing unload. - conf.setLoadBalancerInFlightServiceUnitStateWaitingTimeInMillis(5 * 1000); - conf.setLoadBalancerServiceUnitStateMonitorIntervalInSeconds(1); + private static void initConfig(ServiceConfiguration conf){ conf.setForceDeleteNamespaceAllowed(true); conf.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED); conf.setAllowAutoTopicCreation(true); @@ -179,15 +178,18 @@ public void setup() throws Exception { conf.setLoadBalancerSheddingEnabled(false); conf.setLoadBalancerDebugModeEnabled(true); conf.setTopicLevelPoliciesEnabled(true); + } + + @BeforeClass + @Override + public void setup() throws Exception { + // Set the inflight state waiting time and ownership monitor delay time to 5 seconds to avoid + // stuck when doing unload. + initConfig(conf); super.internalSetup(conf); pulsar1 = pulsar; ServiceConfiguration defaultConf = getDefaultConf(); - defaultConf.setAllowAutoTopicCreation(true); - defaultConf.setForceDeleteNamespaceAllowed(true); - defaultConf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); - defaultConf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); - defaultConf.setLoadBalancerSheddingEnabled(false); - defaultConf.setTopicLevelPoliciesEnabled(true); + initConfig(defaultConf); additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf); pulsar2 = additionalPulsarTestContext.getPulsarService(); @@ -213,10 +215,8 @@ public void setup() throws Exception { @Override @AfterClass(alwaysRun = true) protected void cleanup() throws Exception { - pulsar1 = null; - pulsar2.close(); - super.internalCleanup(); this.additionalPulsarTestContext.close(); + super.internalCleanup(); } @BeforeMethod(alwaysRun = true) @@ -256,9 +256,6 @@ public void testAssign() throws Exception { Optional brokerLookupData1 = secondaryLoadManager.assign(Optional.empty(), bundle).get(); assertEquals(brokerLookupData, brokerLookupData1); - verify(primaryLoadManager, times(1)).getBrokerSelectionStrategy(); - verify(secondaryLoadManager, times(0)).getBrokerSelectionStrategy(); - Optional lookupResult = pulsar2.getNamespaceService() .getBrokerServiceUrlAsync(topicName, null).get(); assertTrue(lookupResult.isPresent()); @@ -314,7 +311,7 @@ public String name() { public CompletableFuture> filterAsync(Map brokers, ServiceUnitId serviceUnit, LoadManagerContext context) { - brokers.remove(pulsar1.getLookupServiceAddress()); + brokers.remove(pulsar1.getBrokerId()); return CompletableFuture.completedFuture(brokers); } @@ -399,10 +396,10 @@ public boolean test(NamespaceBundle namespaceBundle) { }); - String dstBrokerUrl = pulsar1.getLookupServiceAddress(); + String dstBrokerUrl = pulsar1.getBrokerId(); String dstBrokerServiceUrl; if (broker.equals(pulsar1.getBrokerServiceUrl())) { - dstBrokerUrl = pulsar2.getLookupServiceAddress(); + dstBrokerUrl = pulsar2.getBrokerId(); dstBrokerServiceUrl = pulsar2.getBrokerServiceUrl(); } else { dstBrokerServiceUrl = pulsar1.getBrokerServiceUrl(); @@ -482,10 +479,10 @@ public void testTransferClientReconnectionWithoutLookup(TopicDomain topicDomain, final String dstBrokerUrl; final String dstBrokerServiceUrl; if (broker.equals(pulsar1.getBrokerServiceUrl())) { - dstBrokerUrl = pulsar2.getLookupServiceAddress(); + dstBrokerUrl = pulsar2.getBrokerId(); dstBrokerServiceUrl = pulsar2.getBrokerServiceUrl(); } else { - dstBrokerUrl = pulsar1.getLookupServiceAddress(); + dstBrokerUrl = pulsar1.getBrokerId(); dstBrokerServiceUrl = pulsar1.getBrokerServiceUrl(); } checkOwnershipState(broker, bundle); @@ -749,7 +746,8 @@ public void testSplitBundleWithSpecificPositionAdminAPI() throws Exception { "specified_positions_divide", List.of(bundleRanges.get(0), bundleRanges.get(1), splitPosition)); BundlesData bundlesData = admin.namespaces().getBundles(namespace); - assertEquals(bundlesData.getNumBundles(), numBundles + 1); + Awaitility.waitAtMost(15, TimeUnit.SECONDS) + .untilAsserted(() -> assertEquals(bundlesData.getNumBundles(), numBundles + 1)); String lowBundle = String.format("0x%08x", bundleRanges.get(0)); String midBundle = String.format("0x%08x", splitPosition); String highBundle = String.format("0x%08x", bundleRanges.get(1)); @@ -763,10 +761,8 @@ public void testDeleteNamespaceBundle() throws Exception { admin.namespaces().createNamespace(namespace, 3); TopicName topicName = TopicName.get(namespace + "/test-delete-namespace-bundle"); - - Awaitility.await() - .atMost(30, TimeUnit.SECONDS) + .atMost(15, TimeUnit.SECONDS) .ignoreExceptions() .untilAsserted(() -> { NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); @@ -779,7 +775,10 @@ public void testDeleteNamespaceBundle() throws Exception { assertFalse(primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); }); - admin.namespaces().deleteNamespace(namespace, true); + Awaitility.await() + .atMost(15, TimeUnit.SECONDS) + .ignoreExceptions() + .untilAsserted(() -> admin.namespaces().deleteNamespace(namespace, true)); } @Test(timeOut = 30 * 1000) @@ -826,13 +825,13 @@ public void testMoreThenOneFilter() throws Exception { TopicName topicName = topicAndBundle.getLeft(); NamespaceBundle bundle = topicAndBundle.getRight(); - String lookupServiceAddress1 = pulsar1.getLookupServiceAddress(); + String brokerId1 = pulsar1.getBrokerId(); doReturn(List.of(new MockBrokerFilter() { @Override public CompletableFuture> filterAsync(Map brokers, ServiceUnitId serviceUnit, LoadManagerContext context) { - brokers.remove(lookupServiceAddress1); + brokers.remove(brokerId1); return CompletableFuture.completedFuture(brokers); } },new MockBrokerFilter() { @@ -904,12 +903,12 @@ public void testDeployAndRollbackLoadManager() throws Exception { // Test lookup heartbeat namespace's topic for (PulsarService pulsar : pulsarServices) { assertLookupHeartbeatOwner(pulsarService, - pulsar.getLookupServiceAddress(), pulsar.getBrokerServiceUrl()); + pulsar.getBrokerId(), pulsar.getBrokerServiceUrl()); } // Test lookup SLA namespace's topic for (PulsarService pulsar : pulsarServices) { assertLookupSLANamespaceOwner(pulsarService, - pulsar.getLookupServiceAddress(), pulsar.getBrokerServiceUrl()); + pulsar.getBrokerId(), pulsar.getBrokerServiceUrl()); } } @@ -966,12 +965,12 @@ public void testDeployAndRollbackLoadManager() throws Exception { // Test lookup heartbeat namespace's topic for (PulsarService pulsar : pulsarServices) { assertLookupHeartbeatOwner(pulsarService, - pulsar.getLookupServiceAddress(), pulsar.getBrokerServiceUrl()); + pulsar.getBrokerId(), pulsar.getBrokerServiceUrl()); } // Test lookup SLA namespace's topic for (PulsarService pulsar : pulsarServices) { assertLookupSLANamespaceOwner(pulsarService, - pulsar.getLookupServiceAddress(), pulsar.getBrokerServiceUrl()); + pulsar.getBrokerId(), pulsar.getBrokerServiceUrl()); } } } @@ -979,25 +978,25 @@ public void testDeployAndRollbackLoadManager() throws Exception { } private void assertLookupHeartbeatOwner(PulsarService pulsar, - String lookupServiceAddress, + String brokerId, String expectedBrokerServiceUrl) throws Exception { NamespaceName heartbeatNamespaceV1 = - getHeartbeatNamespace(lookupServiceAddress, pulsar.getConfiguration()); + getHeartbeatNamespace(brokerId, pulsar.getConfiguration()); String heartbeatV1Topic = heartbeatNamespaceV1.getPersistentTopicName("test"); assertEquals(pulsar.getAdminClient().lookups().lookupTopic(heartbeatV1Topic), expectedBrokerServiceUrl); NamespaceName heartbeatNamespaceV2 = - getHeartbeatNamespaceV2(lookupServiceAddress, pulsar.getConfiguration()); + getHeartbeatNamespaceV2(brokerId, pulsar.getConfiguration()); String heartbeatV2Topic = heartbeatNamespaceV2.getPersistentTopicName("test"); assertEquals(pulsar.getAdminClient().lookups().lookupTopic(heartbeatV2Topic), expectedBrokerServiceUrl); } private void assertLookupSLANamespaceOwner(PulsarService pulsar, - String lookupServiceAddress, + String brokerId, String expectedBrokerServiceUrl) throws Exception { - NamespaceName slaMonitorNamespace = getSLAMonitorNamespace(lookupServiceAddress, pulsar.getConfiguration()); + NamespaceName slaMonitorNamespace = getSLAMonitorNamespace(brokerId, pulsar.getConfiguration()); String slaMonitorTopic = slaMonitorNamespace.getPersistentTopicName("test"); String result = pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic); log.info("Topic {} Lookup result: {}", slaMonitorTopic, result); @@ -1005,7 +1004,7 @@ private void assertLookupSLANamespaceOwner(PulsarService pulsar, assertEquals(result, expectedBrokerServiceUrl); } - @Test + @Test(priority = 10) public void testTopBundlesLoadDataStoreTableViewFromChannelOwner() throws Exception { var topBundlesLoadDataStorePrimary = (LoadDataStore) FieldUtils.readDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", true); @@ -1073,7 +1072,6 @@ public void testRoleChange() throws Exception { reset(); return null; }).when(topBundlesLoadDataStorePrimarySpy).closeTableView(); - FieldUtils.writeDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", topBundlesLoadDataStorePrimarySpy, true); var topBundlesLoadDataStoreSecondary = (LoadDataStore) FieldUtils.readDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", true); @@ -1096,36 +1094,65 @@ public void testRoleChange() throws Exception { reset(); return null; }).when(topBundlesLoadDataStoreSecondarySpy).closeTableView(); - FieldUtils.writeDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", topBundlesLoadDataStoreSecondarySpy, true); - if (channel1.isChannelOwnerAsync().get(5, TimeUnit.SECONDS)) { - primaryLoadManager.playFollower(); // close 3 times - primaryLoadManager.playFollower(); // close 1 time - secondaryLoadManager.playLeader(); - secondaryLoadManager.playLeader(); - primaryLoadManager.playLeader(); // close 3 times and open 3 times - primaryLoadManager.playLeader(); // close 1 time and open 1 time, - secondaryLoadManager.playFollower(); - secondaryLoadManager.playFollower(); - } else { - primaryLoadManager.playLeader(); - primaryLoadManager.playLeader(); - secondaryLoadManager.playFollower(); - secondaryLoadManager.playFollower(); - primaryLoadManager.playFollower(); + try { + FieldUtils.writeDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", + topBundlesLoadDataStorePrimarySpy, true); + FieldUtils.writeDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", + topBundlesLoadDataStoreSecondarySpy, true); + + + if (channel1.isChannelOwnerAsync().get(5, TimeUnit.SECONDS)) { + primaryLoadManager.playLeader(); + secondaryLoadManager.playFollower(); + verify(topBundlesLoadDataStorePrimarySpy, times(3)).startTableView(); + verify(topBundlesLoadDataStorePrimarySpy, times(5)).closeTableView(); + verify(topBundlesLoadDataStoreSecondarySpy, times(0)).startTableView(); + verify(topBundlesLoadDataStoreSecondarySpy, times(3)).closeTableView(); + } else { + primaryLoadManager.playFollower(); + secondaryLoadManager.playLeader(); + verify(topBundlesLoadDataStoreSecondarySpy, times(3)).startTableView(); + verify(topBundlesLoadDataStoreSecondarySpy, times(5)).closeTableView(); + verify(topBundlesLoadDataStorePrimarySpy, times(0)).startTableView(); + verify(topBundlesLoadDataStorePrimarySpy, times(3)).closeTableView(); + } + primaryLoadManager.playFollower(); - secondaryLoadManager.playLeader(); - secondaryLoadManager.playLeader(); - } + secondaryLoadManager.playFollower(); + if (channel1.isChannelOwnerAsync().get(5, TimeUnit.SECONDS)) { + assertEquals(ExtensibleLoadManagerImpl.Role.Leader, + FieldUtils.readDeclaredField(primaryLoadManager, "role", true)); + assertEquals(ExtensibleLoadManagerImpl.Role.Follower, + FieldUtils.readDeclaredField(secondaryLoadManager, "role", true)); + } else { + assertEquals(ExtensibleLoadManagerImpl.Role.Follower, + FieldUtils.readDeclaredField(primaryLoadManager, "role", true)); + assertEquals(ExtensibleLoadManagerImpl.Role.Leader, + FieldUtils.readDeclaredField(secondaryLoadManager, "role", true)); + } - verify(topBundlesLoadDataStorePrimarySpy, times(4)).startTableView(); - verify(topBundlesLoadDataStorePrimarySpy, times(8)).closeTableView(); - verify(topBundlesLoadDataStoreSecondarySpy, times(4)).startTableView(); - verify(topBundlesLoadDataStoreSecondarySpy, times(8)).closeTableView(); + primaryLoadManager.playLeader(); + secondaryLoadManager.playLeader(); - FieldUtils.writeDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", topBundlesLoadDataStorePrimary, true); - FieldUtils.writeDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", topBundlesLoadDataStoreSecondary, true); + if (channel1.isChannelOwnerAsync().get(5, TimeUnit.SECONDS)) { + assertEquals(ExtensibleLoadManagerImpl.Role.Leader, + FieldUtils.readDeclaredField(primaryLoadManager, "role", true)); + assertEquals(ExtensibleLoadManagerImpl.Role.Follower, + FieldUtils.readDeclaredField(secondaryLoadManager, "role", true)); + } else { + assertEquals(ExtensibleLoadManagerImpl.Role.Follower, + FieldUtils.readDeclaredField(primaryLoadManager, "role", true)); + assertEquals(ExtensibleLoadManagerImpl.Role.Leader, + FieldUtils.readDeclaredField(secondaryLoadManager, "role", true)); + } + } finally { + FieldUtils.writeDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", + topBundlesLoadDataStorePrimary, true); + FieldUtils.writeDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", + topBundlesLoadDataStoreSecondary, true); + } } @Test @@ -1338,7 +1365,7 @@ public void testDisableBroker() throws Exception { NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); if (!pulsar3.getBrokerServiceUrl().equals(lookupResult1)) { admin.namespaces().unloadNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange(), - pulsar3.getLookupServiceAddress()); + pulsar3.getBrokerId()); lookupResult1 = pulsar2.getAdminClient().lookups().lookupTopic(topic); } String lookupResult2 = pulsar1.getAdminClient().lookups().lookupTopic(topic); @@ -1405,20 +1432,20 @@ public void testListTopic() throws Exception { @Test(timeOut = 30 * 1000, priority = -1) public void testGetOwnedServiceUnitsAndGetOwnedNamespaceStatus() throws Exception { NamespaceName heartbeatNamespacePulsar1V1 = - getHeartbeatNamespace(pulsar1.getLookupServiceAddress(), pulsar1.getConfiguration()); + getHeartbeatNamespace(pulsar1.getBrokerId(), pulsar1.getConfiguration()); NamespaceName heartbeatNamespacePulsar1V2 = - NamespaceService.getHeartbeatNamespaceV2(pulsar1.getLookupServiceAddress(), pulsar1.getConfiguration()); + NamespaceService.getHeartbeatNamespaceV2(pulsar1.getBrokerId(), pulsar1.getConfiguration()); NamespaceName heartbeatNamespacePulsar2V1 = - getHeartbeatNamespace(pulsar2.getLookupServiceAddress(), pulsar2.getConfiguration()); + getHeartbeatNamespace(pulsar2.getBrokerId(), pulsar2.getConfiguration()); NamespaceName heartbeatNamespacePulsar2V2 = - NamespaceService.getHeartbeatNamespaceV2(pulsar2.getLookupServiceAddress(), pulsar2.getConfiguration()); + NamespaceService.getHeartbeatNamespaceV2(pulsar2.getBrokerId(), pulsar2.getConfiguration()); NamespaceName slaMonitorNamespacePulsar1 = - getSLAMonitorNamespace(pulsar1.getLookupServiceAddress(), pulsar1.getConfiguration()); + getSLAMonitorNamespace(pulsar1.getBrokerId(), pulsar1.getConfiguration()); NamespaceName slaMonitorNamespacePulsar2 = - getSLAMonitorNamespace(pulsar2.getLookupServiceAddress(), pulsar2.getConfiguration()); + getSLAMonitorNamespace(pulsar2.getBrokerId(), pulsar2.getConfiguration()); NamespaceBundle bundle1 = pulsar1.getNamespaceService().getNamespaceBundleFactory() .getFullBundle(heartbeatNamespacePulsar1V1); @@ -1448,9 +1475,9 @@ public void testGetOwnedServiceUnitsAndGetOwnedNamespaceStatus() throws Exceptio assertTrue(ownedServiceUnitsByPulsar2.contains(bundle4)); assertTrue(ownedServiceUnitsByPulsar2.contains(slaBundle2)); Map ownedNamespacesByPulsar1 = - admin.brokers().getOwnedNamespaces(conf.getClusterName(), pulsar1.getLookupServiceAddress()); + admin.brokers().getOwnedNamespaces(conf.getClusterName(), pulsar1.getBrokerId()); Map ownedNamespacesByPulsar2 = - admin.brokers().getOwnedNamespaces(conf.getClusterName(), pulsar2.getLookupServiceAddress()); + admin.brokers().getOwnedNamespaces(conf.getClusterName(), pulsar2.getBrokerId()); assertTrue(ownedNamespacesByPulsar1.containsKey(bundle1.toString())); assertTrue(ownedNamespacesByPulsar1.containsKey(bundle2.toString())); assertTrue(ownedNamespacesByPulsar1.containsKey(slaBundle1.toString())); @@ -1482,7 +1509,7 @@ private void assertOwnedServiceUnits( assertTrue(ownedBundles.contains(bundle)); }); Map ownedNamespaces = - admin.brokers().getOwnedNamespaces(conf.getClusterName(), pulsar.getLookupServiceAddress()); + admin.brokers().getOwnedNamespaces(conf.getClusterName(), pulsar.getBrokerId()); assertTrue(ownedNamespaces.containsKey(bundle.toString())); NamespaceOwnershipStatus status = ownedNamespaces.get(bundle.toString()); assertTrue(status.is_active); @@ -1516,6 +1543,47 @@ public void testHealthcheck() throws PulsarAdminException { admin.brokers().healthcheck(TopicVersion.V2); } + @Test(timeOut = 30 * 1000) + public void compactionScheduleTest() { + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(30, TimeUnit.SECONDS) + .ignoreExceptions() + .untilAsserted(() -> { // wait until true + primaryLoadManager.monitor(); + secondaryLoadManager.monitor(); + var threshold = admin.topicPolicies() + .getCompactionThreshold(ServiceUnitStateChannelImpl.TOPIC, false); + AssertJUnit.assertEquals(5 * 1024 * 1024, threshold == null ? 0 : threshold.longValue()); + }); + } + + @Test(timeOut = 10 * 1000) + public void unloadTimeoutCheckTest() + throws Exception { + Pair topicAndBundle = getBundleIsNotOwnByChangeEventTopic("unload-timeout"); + String topic = topicAndBundle.getLeft().toString(); + var bundle = topicAndBundle.getRight().toString(); + var releasing = new ServiceUnitStateData(Releasing, pulsar2.getBrokerId(), pulsar1.getBrokerId(), 1); + overrideTableView(channel1, bundle, releasing); + var topicFuture = pulsar1.getBrokerService().getOrCreateTopic(topic); + + + try { + topicFuture.get(1, TimeUnit.SECONDS); + } catch (Exception e) { + log.info("getOrCreateTopic failed", e); + if (!(e.getCause() instanceof BrokerServiceException.ServiceUnitNotReadyException && e.getMessage() + .contains("Please redo the lookup"))) { + fail(); + } + } + + pulsar1.getBrokerService() + .unloadServiceUnit(topicAndBundle.getRight(), true, true, 5, + TimeUnit.SECONDS).get(2, TimeUnit.SECONDS); + } + private static abstract class MockBrokerFilter implements BrokerFilter { @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java index f99a167ff4883..fe8387710eeae 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java @@ -39,6 +39,7 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertThrows; +import static org.testng.Assert.expectThrows; import static org.testng.AssertJUnit.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; @@ -86,7 +87,6 @@ import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.TypedMessageBuilder; import org.apache.pulsar.client.impl.TableViewImpl; import org.apache.pulsar.common.policies.data.TopicType; @@ -108,8 +108,8 @@ public class ServiceUnitStateChannelTest extends MockedPulsarServiceBaseTest { private PulsarService pulsar2; private ServiceUnitStateChannel channel1; private ServiceUnitStateChannel channel2; - private String lookupServiceAddress1; - private String lookupServiceAddress2; + private String brokerId1; + private String brokerId2; private String bundle; private String bundle1; private String bundle2; @@ -157,10 +157,10 @@ protected void setup() throws Exception { channel2 = createChannel(pulsar2); channel2.start(); - lookupServiceAddress1 = (String) - FieldUtils.readDeclaredField(channel1, "lookupServiceAddress", true); - lookupServiceAddress2 = (String) - FieldUtils.readDeclaredField(channel2, "lookupServiceAddress", true); + brokerId1 = (String) + FieldUtils.readDeclaredField(channel1, "brokerId", true); + brokerId2 = (String) + FieldUtils.readDeclaredField(channel2, "brokerId", true); bundle = "public/default/0x00000000_0xffffffff"; bundle1 = "public/default/0x00000000_0xfffffff0"; @@ -220,7 +220,7 @@ public void channelOwnerTest() throws Exception { assertEquals(newChannelOwner1, newChannelOwner2); assertNotEquals(channelOwner1, newChannelOwner1); - if (newChannelOwner1.equals(Optional.of(lookupServiceAddress1))) { + if (newChannelOwner1.equals(Optional.of(brokerId1))) { assertTrue(channel1.isChannelOwnerAsync().get(2, TimeUnit.SECONDS)); assertFalse(channel2.isChannelOwnerAsync().get(2, TimeUnit.SECONDS)); } else { @@ -305,7 +305,7 @@ private int validateChannelStart(ServiceUnitStateChannelImpl channel) } } try { - channel.publishAssignEventAsync(bundle, lookupServiceAddress1).get(2, TimeUnit.SECONDS); + channel.publishAssignEventAsync(bundle, brokerId1).get(2, TimeUnit.SECONDS); } catch (ExecutionException e) { if (e.getCause() instanceof IllegalStateException) { errorCnt++; @@ -313,7 +313,7 @@ private int validateChannelStart(ServiceUnitStateChannelImpl channel) } try { channel.publishUnloadEventAsync( - new Unload(lookupServiceAddress1, bundle, Optional.of(lookupServiceAddress2))) + new Unload(brokerId1, bundle, Optional.of(brokerId2))) .get(2, TimeUnit.SECONDS); } catch (ExecutionException e) { if (e.getCause() instanceof IllegalStateException) { @@ -321,7 +321,7 @@ private int validateChannelStart(ServiceUnitStateChannelImpl channel) } } try { - Split split = new Split(bundle, lookupServiceAddress1, Map.of( + Split split = new Split(bundle, brokerId1, Map.of( childBundle1Range, Optional.empty(), childBundle2Range, Optional.empty())); channel.publishSplitEventAsync(split) .get(2, TimeUnit.SECONDS); @@ -333,23 +333,6 @@ private int validateChannelStart(ServiceUnitStateChannelImpl channel) return errorCnt; } - @Test(priority = 1) - public void compactionScheduleTest() { - - Awaitility.await() - .pollInterval(200, TimeUnit.MILLISECONDS) - .atMost(5, TimeUnit.SECONDS) - .untilAsserted(() -> { // wait until true - try { - var threshold = admin.topicPolicies() - .getCompactionThreshold(ServiceUnitStateChannelImpl.TOPIC, false).longValue(); - assertEquals(5 * 1024 * 1024, threshold); - } catch (Exception e) { - ; - } - }); - } - @Test(priority = 2) public void assignmentTest() throws ExecutionException, InterruptedException, IllegalAccessException, TimeoutException { @@ -362,8 +345,8 @@ public void assignmentTest() assertTrue(owner1.get().isEmpty()); assertTrue(owner2.get().isEmpty()); - var assigned1 = channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); - var assigned2 = channel2.publishAssignEventAsync(bundle, lookupServiceAddress2); + var assigned1 = channel1.publishAssignEventAsync(bundle, brokerId1); + var assigned2 = channel2.publishAssignEventAsync(bundle, brokerId2); assertNotNull(assigned1); assertNotNull(assigned2); waitUntilOwnerChanges(channel1, bundle, null); @@ -372,8 +355,8 @@ public void assignmentTest() String assignedAddr2 = assigned2.get(5, TimeUnit.SECONDS); assertEquals(assignedAddr1, assignedAddr2); - assertTrue(assignedAddr1.equals(lookupServiceAddress1) - || assignedAddr1.equals(lookupServiceAddress2), assignedAddr1); + assertTrue(assignedAddr1.equals(brokerId1) + || assignedAddr1.equals(brokerId2), assignedAddr1); var ownerAddr1 = channel1.getOwnerAsync(bundle).get(); var ownerAddr2 = channel2.getOwnerAsync(bundle).get(); @@ -414,13 +397,13 @@ public void assignmentTestWhenOneAssignmentFails() assertTrue(owner1.get().isEmpty()); assertTrue(owner2.get().isEmpty()); - var owner3 = channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); - var owner4 = channel2.publishAssignEventAsync(bundle, lookupServiceAddress2); + var owner3 = channel1.publishAssignEventAsync(bundle, brokerId1); + var owner4 = channel2.publishAssignEventAsync(bundle, brokerId2); assertTrue(owner3.isCompletedExceptionally()); assertNotNull(owner4); String ownerAddrOpt2 = owner4.get(5, TimeUnit.SECONDS); - assertEquals(ownerAddrOpt2, lookupServiceAddress2); - waitUntilNewOwner(channel1, bundle, lookupServiceAddress2); + assertEquals(ownerAddrOpt2, brokerId2); + waitUntilNewOwner(channel1, bundle, brokerId2); assertEquals(0, getOwnerRequests1.size()); assertEquals(0, getOwnerRequests2.size()); @@ -438,25 +421,25 @@ public void transferTest() assertTrue(owner2.get().isEmpty()); - channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); - waitUntilNewOwner(channel1, bundle, lookupServiceAddress1); - waitUntilNewOwner(channel2, bundle, lookupServiceAddress1); + channel1.publishAssignEventAsync(bundle, brokerId1); + waitUntilNewOwner(channel1, bundle, brokerId1); + waitUntilNewOwner(channel2, bundle, brokerId1); var ownerAddr1 = channel1.getOwnerAsync(bundle).get(); var ownerAddr2 = channel2.getOwnerAsync(bundle).get(); assertEquals(ownerAddr1, ownerAddr2); - assertEquals(ownerAddr1, Optional.of(lookupServiceAddress1)); + assertEquals(ownerAddr1, Optional.of(brokerId1)); - Unload unload = new Unload(lookupServiceAddress1, bundle, Optional.of(lookupServiceAddress2)); + Unload unload = new Unload(brokerId1, bundle, Optional.of(brokerId2)); channel1.publishUnloadEventAsync(unload); - waitUntilNewOwner(channel1, bundle, lookupServiceAddress2); - waitUntilNewOwner(channel2, bundle, lookupServiceAddress2); + waitUntilNewOwner(channel1, bundle, brokerId2); + waitUntilNewOwner(channel2, bundle, brokerId2); ownerAddr1 = channel1.getOwnerAsync(bundle).get(5, TimeUnit.SECONDS); ownerAddr2 = channel2.getOwnerAsync(bundle).get(5, TimeUnit.SECONDS); assertEquals(ownerAddr1, ownerAddr2); - assertEquals(ownerAddr1, Optional.of(lookupServiceAddress2)); + assertEquals(ownerAddr1, Optional.of(brokerId2)); validateHandlerCounters(channel1, 2, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0); validateHandlerCounters(channel2, 2, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0); @@ -473,14 +456,14 @@ public void transferTestWhenDestBrokerFails() assertEquals(0, getOwnerRequests1.size()); assertEquals(0, getOwnerRequests2.size()); - channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); - waitUntilNewOwner(channel1, bundle, lookupServiceAddress1); - waitUntilNewOwner(channel2, bundle, lookupServiceAddress1); + channel1.publishAssignEventAsync(bundle, brokerId1); + waitUntilNewOwner(channel1, bundle, brokerId1); + waitUntilNewOwner(channel2, bundle, brokerId1); var ownerAddr1 = channel1.getOwnerAsync(bundle).get(); var ownerAddr2 = channel2.getOwnerAsync(bundle).get(); assertEquals(ownerAddr1, ownerAddr2); - assertEquals(ownerAddr1, Optional.of(lookupServiceAddress1)); + assertEquals(ownerAddr1, Optional.of(brokerId1)); var producer = (Producer) FieldUtils.readDeclaredField(channel1, "producer", true); @@ -496,7 +479,7 @@ public void transferTestWhenDestBrokerFails() "inFlightStateWaitingTimeInMillis", 3 * 1000, true); FieldUtils.writeDeclaredField(channel2, "inFlightStateWaitingTimeInMillis", 3 * 1000, true); - Unload unload = new Unload(lookupServiceAddress1, bundle, Optional.of(lookupServiceAddress2)); + Unload unload = new Unload(brokerId1, bundle, Optional.of(brokerId2)); channel1.publishUnloadEventAsync(unload); // channel1 is broken. the ownership transfer won't be complete. waitUntilState(channel1, bundle); @@ -504,23 +487,21 @@ public void transferTestWhenDestBrokerFails() var owner1 = channel1.getOwnerAsync(bundle); var owner2 = channel2.getOwnerAsync(bundle); - assertFalse(owner1.isDone()); + assertTrue(owner1.isDone()); + assertEquals(brokerId2, owner1.get().get()); assertFalse(owner2.isDone()); - assertEquals(1, getOwnerRequests1.size()); + assertEquals(0, getOwnerRequests1.size()); assertEquals(1, getOwnerRequests2.size()); // In 10 secs, the getOwnerAsync requests(lookup requests) should time out. - Awaitility.await().atMost(10, TimeUnit.SECONDS) - .untilAsserted(() -> assertTrue(owner1.isCompletedExceptionally())); Awaitility.await().atMost(10, TimeUnit.SECONDS) .untilAsserted(() -> assertTrue(owner2.isCompletedExceptionally())); - assertEquals(0, getOwnerRequests1.size()); assertEquals(0, getOwnerRequests2.size()); // recovered, check the monitor update state : Assigned -> Owned - doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress1))) + doReturn(CompletableFuture.completedFuture(Optional.of(brokerId1))) .when(loadManager).selectAsync(any(), any()); FieldUtils.writeDeclaredField(channel2, "producer", producer, true); FieldUtils.writeDeclaredField(channel1, @@ -529,18 +510,18 @@ public void transferTestWhenDestBrokerFails() "inFlightStateWaitingTimeInMillis", 1 , true); ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( - List.of(lookupServiceAddress1, lookupServiceAddress2)); + List.of(brokerId1, brokerId2)); ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( - List.of(lookupServiceAddress1, lookupServiceAddress2)); + List.of(brokerId1, brokerId2)); - waitUntilNewOwner(channel1, bundle, lookupServiceAddress1); - waitUntilNewOwner(channel2, bundle, lookupServiceAddress1); + waitUntilNewOwner(channel1, bundle, brokerId1); + waitUntilNewOwner(channel2, bundle, brokerId1); ownerAddr1 = channel1.getOwnerAsync(bundle).get(); ownerAddr2 = channel2.getOwnerAsync(bundle).get(); assertEquals(ownerAddr1, ownerAddr2); - assertEquals(ownerAddr1, Optional.of(lookupServiceAddress1)); + assertEquals(ownerAddr1, Optional.of(brokerId1)); var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; validateMonitorCounters(leader, @@ -561,13 +542,13 @@ public void transferTestWhenDestBrokerFails() @Test(priority = 6) public void splitAndRetryTest() throws Exception { - channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); - waitUntilNewOwner(channel1, bundle, lookupServiceAddress1); - waitUntilNewOwner(channel2, bundle, lookupServiceAddress1); + channel1.publishAssignEventAsync(bundle, brokerId1); + waitUntilNewOwner(channel1, bundle, brokerId1); + waitUntilNewOwner(channel2, bundle, brokerId1); var ownerAddr1 = channel1.getOwnerAsync(bundle).get(); var ownerAddr2 = channel2.getOwnerAsync(bundle).get(); - assertEquals(ownerAddr1, Optional.of(lookupServiceAddress1)); - assertEquals(ownerAddr2, Optional.of(lookupServiceAddress1)); + assertEquals(ownerAddr1, Optional.of(brokerId1)); + assertEquals(ownerAddr2, Optional.of(brokerId1)); assertTrue(ownerAddr1.isPresent()); NamespaceService namespaceService = pulsar1.getNamespaceService(); @@ -608,14 +589,14 @@ public void splitAndRetryTest() throws Exception { - waitUntilNewOwner(channel1, childBundle11, lookupServiceAddress1); - waitUntilNewOwner(channel1, childBundle12, lookupServiceAddress1); - waitUntilNewOwner(channel2, childBundle11, lookupServiceAddress1); - waitUntilNewOwner(channel2, childBundle12, lookupServiceAddress1); - assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(childBundle11).get()); - assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(childBundle12).get()); - assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle11).get()); - assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle12).get()); + waitUntilNewOwner(channel1, childBundle11, brokerId1); + waitUntilNewOwner(channel1, childBundle12, brokerId1); + waitUntilNewOwner(channel2, childBundle11, brokerId1); + waitUntilNewOwner(channel2, childBundle12, brokerId1); + assertEquals(Optional.of(brokerId1), channel1.getOwnerAsync(childBundle11).get()); + assertEquals(Optional.of(brokerId1), channel1.getOwnerAsync(childBundle12).get()); + assertEquals(Optional.of(brokerId1), channel2.getOwnerAsync(childBundle11).get()); + assertEquals(Optional.of(brokerId1), channel2.getOwnerAsync(childBundle12).get()); // try the monitor and check the monitor moves `Deleted` -> `Init` @@ -630,9 +611,9 @@ public void splitAndRetryTest() throws Exception { "stateTombstoneDelayTimeInMillis", 1, true); ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( - List.of(lookupServiceAddress1, lookupServiceAddress2)); + List.of(brokerId1, brokerId2)); ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( - List.of(lookupServiceAddress1, lookupServiceAddress2)); + List.of(brokerId1, brokerId2)); waitUntilState(channel1, bundle, Init); waitUntilState(channel2, bundle, Init); @@ -726,7 +707,7 @@ public void handleBrokerDeletionEventTest() String leader = channel1.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); String leader2 = channel2.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); assertEquals(leader, leader2); - if (leader.equals(lookupServiceAddress2)) { + if (leader.equals(brokerId2)) { leaderChannel = channel2; followerChannel = channel1; var tmp = followerCleanupJobsTmp; @@ -742,12 +723,12 @@ public void handleBrokerDeletionEventTest() var owner1 = channel1.getOwnerAsync(bundle1); var owner2 = channel2.getOwnerAsync(bundle2); - doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))) + doReturn(CompletableFuture.completedFuture(Optional.of(brokerId2))) .when(loadManager).selectAsync(any(), any()); assertTrue(owner1.get().isEmpty()); assertTrue(owner2.get().isEmpty()); - String broker = lookupServiceAddress1; + String broker = brokerId1; channel1.publishAssignEventAsync(bundle1, broker); channel2.publishAssignEventAsync(bundle2, broker); @@ -757,9 +738,9 @@ public void handleBrokerDeletionEventTest() waitUntilNewOwner(channel2, bundle2, broker); // Verify to transfer the ownership to the other broker. - channel1.publishUnloadEventAsync(new Unload(broker, bundle1, Optional.of(lookupServiceAddress2))); - waitUntilNewOwner(channel1, bundle1, lookupServiceAddress2); - waitUntilNewOwner(channel2, bundle1, lookupServiceAddress2); + channel1.publishUnloadEventAsync(new Unload(broker, bundle1, Optional.of(brokerId2))); + waitUntilNewOwner(channel1, bundle1, brokerId2); + waitUntilNewOwner(channel2, bundle1, brokerId2); // test stable metadata state leaderChannel.handleMetadataSessionEvent(SessionReestablished); @@ -770,13 +751,13 @@ public void handleBrokerDeletionEventTest() System.currentTimeMillis() - (MAX_CLEAN_UP_DELAY_TIME_IN_SECS * 1000 + 1000), true); leaderChannel.handleBrokerRegistrationEvent(broker, NotificationType.Deleted); followerChannel.handleBrokerRegistrationEvent(broker, NotificationType.Deleted); - leaderChannel.handleBrokerRegistrationEvent(lookupServiceAddress2, NotificationType.Deleted); - followerChannel.handleBrokerRegistrationEvent(lookupServiceAddress2, NotificationType.Deleted); + leaderChannel.handleBrokerRegistrationEvent(brokerId2, NotificationType.Deleted); + followerChannel.handleBrokerRegistrationEvent(brokerId2, NotificationType.Deleted); - waitUntilNewOwner(channel1, bundle1, lookupServiceAddress2); - waitUntilNewOwner(channel2, bundle1, lookupServiceAddress2); - waitUntilNewOwner(channel1, bundle2, lookupServiceAddress2); - waitUntilNewOwner(channel2, bundle2, lookupServiceAddress2); + waitUntilNewOwner(channel1, bundle1, brokerId2); + waitUntilNewOwner(channel2, bundle1, brokerId2); + waitUntilNewOwner(channel1, bundle2, brokerId2); + waitUntilNewOwner(channel2, bundle2, brokerId2); verify(leaderCleanupJobs, times(1)).computeIfAbsent(eq(broker), any()); verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); @@ -797,8 +778,8 @@ public void handleBrokerDeletionEventTest() // test jittery metadata state - channel1.publishUnloadEventAsync(new Unload(lookupServiceAddress2, bundle1, Optional.of(broker))); - channel1.publishUnloadEventAsync(new Unload(lookupServiceAddress2, bundle2, Optional.of(broker))); + channel1.publishUnloadEventAsync(new Unload(brokerId2, bundle1, Optional.of(broker))); + channel1.publishUnloadEventAsync(new Unload(brokerId2, bundle2, Optional.of(broker))); waitUntilNewOwner(channel1, bundle1, broker); waitUntilNewOwner(channel2, bundle1, broker); waitUntilNewOwner(channel1, bundle2, broker); @@ -870,10 +851,10 @@ public void handleBrokerDeletionEventTest() 1); // finally cleanup - waitUntilNewOwner(channel1, bundle1, lookupServiceAddress2); - waitUntilNewOwner(channel2, bundle1, lookupServiceAddress2); - waitUntilNewOwner(channel1, bundle2, lookupServiceAddress2); - waitUntilNewOwner(channel2, bundle2, lookupServiceAddress2); + waitUntilNewOwner(channel1, bundle1, brokerId2); + waitUntilNewOwner(channel2, bundle1, brokerId2); + waitUntilNewOwner(channel1, bundle2, brokerId2); + waitUntilNewOwner(channel2, bundle2, brokerId2); verify(leaderCleanupJobs, times(3)).computeIfAbsent(eq(broker), any()); verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); @@ -892,8 +873,8 @@ public void handleBrokerDeletionEventTest() 1); // test unstable state - channel1.publishUnloadEventAsync(new Unload(lookupServiceAddress2, bundle1, Optional.of(broker))); - channel1.publishUnloadEventAsync(new Unload(lookupServiceAddress2, bundle2, Optional.of(broker))); + channel1.publishUnloadEventAsync(new Unload(brokerId2, bundle1, Optional.of(broker))); + channel1.publishUnloadEventAsync(new Unload(brokerId2, bundle2, Optional.of(broker))); waitUntilNewOwner(channel1, bundle1, broker); waitUntilNewOwner(channel2, bundle1, broker); waitUntilNewOwner(channel1, bundle2, broker); @@ -929,25 +910,24 @@ public void handleBrokerDeletionEventTest() } @Test(priority = 10) - public void conflictAndCompactionTest() throws ExecutionException, InterruptedException, TimeoutException, - IllegalAccessException, PulsarClientException, PulsarServerException { + public void conflictAndCompactionTest() throws Exception { String bundle = String.format("%s/%s", "public/default", "0x0000000a_0xffffffff"); var owner1 = channel1.getOwnerAsync(bundle); var owner2 = channel2.getOwnerAsync(bundle); assertTrue(owner1.get().isEmpty()); assertTrue(owner2.get().isEmpty()); - var assigned1 = channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); + var assigned1 = channel1.publishAssignEventAsync(bundle, brokerId1); assertNotNull(assigned1); - waitUntilNewOwner(channel1, bundle, lookupServiceAddress1); - waitUntilNewOwner(channel2, bundle, lookupServiceAddress1); + waitUntilNewOwner(channel1, bundle, brokerId1); + waitUntilNewOwner(channel2, bundle, brokerId1); String assignedAddr1 = assigned1.get(5, TimeUnit.SECONDS); - assertEquals(lookupServiceAddress1, assignedAddr1); + assertEquals(brokerId1, assignedAddr1); FieldUtils.writeDeclaredField(channel2, "inFlightStateWaitingTimeInMillis", 3 * 1000, true); - var assigned2 = channel2.publishAssignEventAsync(bundle, lookupServiceAddress2); + var assigned2 = channel2.publishAssignEventAsync(bundle, brokerId2); assertNotNull(assigned2); Exception ex = null; try { @@ -956,33 +936,48 @@ public void conflictAndCompactionTest() throws ExecutionException, InterruptedEx ex = e; } assertNull(ex); - assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(bundle).get()); - assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(bundle).get()); + assertEquals(Optional.of(brokerId1), channel2.getOwnerAsync(bundle).get()); + assertEquals(Optional.of(brokerId1), channel1.getOwnerAsync(bundle).get()); var compactor = spy (pulsar1.getStrategicCompactor()); Field strategicCompactorField = FieldUtils.getDeclaredField(PulsarService.class, "strategicCompactor", true); FieldUtils.writeField(strategicCompactorField, pulsar1, compactor, true); FieldUtils.writeField(strategicCompactorField, pulsar2, compactor, true); - Awaitility.await() - .pollInterval(200, TimeUnit.MILLISECONDS) - .atMost(140, TimeUnit.SECONDS) - .untilAsserted(() -> { - channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); - verify(compactor, times(1)) - .compact(eq(ServiceUnitStateChannelImpl.TOPIC), any()); - }); + + var threshold = admin.topicPolicies() + .getCompactionThreshold(ServiceUnitStateChannelImpl.TOPIC); + admin.topicPolicies() + .setCompactionThreshold(ServiceUnitStateChannelImpl.TOPIC, 0); + + try { + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(140, TimeUnit.SECONDS) + .untilAsserted(() -> { + channel1.publishAssignEventAsync(bundle, brokerId1); + verify(compactor, times(1)) + .compact(eq(ServiceUnitStateChannelImpl.TOPIC), any()); + }); + + + var channel3 = createChannel(pulsar); + channel3.start(); + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> assertEquals( + channel3.getOwnerAsync(bundle).get(), Optional.of(brokerId1))); + channel3.close(); + } finally { + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); + if (threshold != null) { + admin.topicPolicies() + .setCompactionThreshold(ServiceUnitStateChannelImpl.TOPIC, threshold); + } + } - var channel3 = createChannel(pulsar); - channel3.start(); - Awaitility.await() - .pollInterval(200, TimeUnit.MILLISECONDS) - .atMost(5, TimeUnit.SECONDS) - .untilAsserted(() -> assertEquals( - channel3.getOwnerAsync(bundle).get(), Optional.of(lookupServiceAddress1))); - channel3.close(); - FieldUtils.writeDeclaredField(channel2, - "inFlightStateWaitingTimeInMillis", 30 * 1000, true); } @Test(priority = 11) @@ -1024,16 +1019,16 @@ public void ownerLookupCountTests() throws IllegalAccessException { public void unloadTest() throws ExecutionException, InterruptedException, IllegalAccessException { - channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); + channel1.publishAssignEventAsync(bundle, brokerId1); - waitUntilNewOwner(channel1, bundle, lookupServiceAddress1); - waitUntilNewOwner(channel2, bundle, lookupServiceAddress1); + waitUntilNewOwner(channel1, bundle, brokerId1); + waitUntilNewOwner(channel2, bundle, brokerId1); var ownerAddr1 = channel1.getOwnerAsync(bundle).get(); var ownerAddr2 = channel2.getOwnerAsync(bundle).get(); assertEquals(ownerAddr1, ownerAddr2); - assertEquals(ownerAddr1, Optional.of(lookupServiceAddress1)); - Unload unload = new Unload(lookupServiceAddress1, bundle, Optional.empty()); + assertEquals(ownerAddr1, Optional.of(brokerId1)); + Unload unload = new Unload(brokerId1, bundle, Optional.empty()); channel1.publishUnloadEventAsync(unload); @@ -1045,17 +1040,17 @@ public void unloadTest() assertEquals(Optional.empty(), owner1.get()); assertEquals(Optional.empty(), owner2.get()); - channel2.publishAssignEventAsync(bundle, lookupServiceAddress2); + channel2.publishAssignEventAsync(bundle, brokerId2); - waitUntilNewOwner(channel1, bundle, lookupServiceAddress2); - waitUntilNewOwner(channel2, bundle, lookupServiceAddress2); + waitUntilNewOwner(channel1, bundle, brokerId2); + waitUntilNewOwner(channel2, bundle, brokerId2); ownerAddr1 = channel1.getOwnerAsync(bundle).get(); ownerAddr2 = channel2.getOwnerAsync(bundle).get(); assertEquals(ownerAddr1, ownerAddr2); - assertEquals(ownerAddr1, Optional.of(lookupServiceAddress2)); - Unload unload2 = new Unload(lookupServiceAddress2, bundle, Optional.empty()); + assertEquals(ownerAddr1, Optional.of(brokerId2)); + Unload unload2 = new Unload(brokerId2, bundle, Optional.empty()); channel2.publishUnloadEventAsync(unload2); @@ -1074,9 +1069,9 @@ public void unloadTest() "stateTombstoneDelayTimeInMillis", 1, true); ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( - List.of(lookupServiceAddress1, lookupServiceAddress2)); + List.of(brokerId1, brokerId2)); ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( - List.of(lookupServiceAddress1, lookupServiceAddress2)); + List.of(brokerId1, brokerId2)); waitUntilState(channel1, bundle, Init); waitUntilState(channel2, bundle, Init); @@ -1106,7 +1101,7 @@ public void unloadTest() public void assignTestWhenDestBrokerProducerFails() throws ExecutionException, InterruptedException, IllegalAccessException { - Unload unload = new Unload(lookupServiceAddress1, bundle, Optional.empty()); + Unload unload = new Unload(brokerId1, bundle, Optional.empty()); channel1.publishUnloadEventAsync(unload); @@ -1130,21 +1125,19 @@ public void assignTestWhenDestBrokerProducerFails() "inFlightStateWaitingTimeInMillis", 3 * 1000, true); FieldUtils.writeDeclaredField(channel2, "inFlightStateWaitingTimeInMillis", 3 * 1000, true); - doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))) + doReturn(CompletableFuture.completedFuture(Optional.of(brokerId2))) .when(loadManager).selectAsync(any(), any()); - channel1.publishAssignEventAsync(bundle, lookupServiceAddress2); + channel1.publishAssignEventAsync(bundle, brokerId2); // channel1 is broken. the assign won't be complete. waitUntilState(channel1, bundle); waitUntilState(channel2, bundle); var owner1 = channel1.getOwnerAsync(bundle); var owner2 = channel2.getOwnerAsync(bundle); - assertFalse(owner1.isDone()); + assertTrue(owner1.isDone()); assertFalse(owner2.isDone()); // In 10 secs, the getOwnerAsync requests(lookup requests) should time out. - Awaitility.await().atMost(10, TimeUnit.SECONDS) - .untilAsserted(() -> assertTrue(owner1.isCompletedExceptionally())); Awaitility.await().atMost(10, TimeUnit.SECONDS) .untilAsserted(() -> assertTrue(owner2.isCompletedExceptionally())); @@ -1156,18 +1149,18 @@ public void assignTestWhenDestBrokerProducerFails() "inFlightStateWaitingTimeInMillis", 1 , true); ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( - List.of(lookupServiceAddress1, lookupServiceAddress2)); + List.of(brokerId1, brokerId2)); ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( - List.of(lookupServiceAddress1, lookupServiceAddress2)); + List.of(brokerId1, brokerId2)); - waitUntilNewOwner(channel1, bundle, lookupServiceAddress2); - waitUntilNewOwner(channel2, bundle, lookupServiceAddress2); + waitUntilNewOwner(channel1, bundle, brokerId2); + waitUntilNewOwner(channel2, bundle, brokerId2); var ownerAddr1 = channel1.getOwnerAsync(bundle).get(); var ownerAddr2 = channel2.getOwnerAsync(bundle).get(); assertEquals(ownerAddr1, ownerAddr2); - assertEquals(ownerAddr1, Optional.of(lookupServiceAddress2)); + assertEquals(ownerAddr1, Optional.of(brokerId2)); var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; validateMonitorCounters(leader, @@ -1191,20 +1184,20 @@ public void splitTestWhenProducerFails() throws ExecutionException, InterruptedException, IllegalAccessException { - Unload unload = new Unload(lookupServiceAddress1, bundle, Optional.empty()); + Unload unload = new Unload(brokerId1, bundle, Optional.empty()); channel1.publishUnloadEventAsync(unload); waitUntilState(channel1, bundle, Free); waitUntilState(channel2, bundle, Free); - channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); + channel1.publishAssignEventAsync(bundle, brokerId1); waitUntilState(channel1, bundle, Owned); waitUntilState(channel2, bundle, Owned); - assertEquals(lookupServiceAddress1, channel1.getOwnerAsync(bundle).get().get()); - assertEquals(lookupServiceAddress1, channel2.getOwnerAsync(bundle).get().get()); + assertEquals(brokerId1, channel1.getOwnerAsync(bundle).get().get()); + assertEquals(brokerId1, channel2.getOwnerAsync(bundle).get().get()); var producer = (Producer) FieldUtils.readDeclaredField(channel1, "producer", true); @@ -1223,7 +1216,7 @@ public void splitTestWhenProducerFails() // Assert child bundle ownerships in the channels. - Split split = new Split(bundle, lookupServiceAddress1, Map.of( + Split split = new Split(bundle, brokerId1, Map.of( childBundle1Range, Optional.empty(), childBundle2Range, Optional.empty())); channel2.publishSplitEventAsync(split); // channel1 is broken. the split won't be complete. @@ -1270,13 +1263,13 @@ public void testIsOwner() throws IllegalAccessException { assertFalse(owner1); assertFalse(owner2); - owner1 = channel1.isOwner(bundle, lookupServiceAddress2); - owner2 = channel2.isOwner(bundle, lookupServiceAddress1); + owner1 = channel1.isOwner(bundle, brokerId2); + owner2 = channel2.isOwner(bundle, brokerId1); assertFalse(owner1); assertFalse(owner2); - channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); + channel1.publishAssignEventAsync(bundle, brokerId1); owner2 = channel2.isOwner(bundle); assertFalse(owner2); @@ -1289,49 +1282,111 @@ public void testIsOwner() throws IllegalAccessException { assertTrue(owner1); assertFalse(owner2); - owner1 = channel1.isOwner(bundle, lookupServiceAddress1); - owner2 = channel2.isOwner(bundle, lookupServiceAddress2); + owner1 = channel1.isOwner(bundle, brokerId1); + owner2 = channel2.isOwner(bundle, brokerId2); assertTrue(owner1); assertFalse(owner2); - owner1 = channel2.isOwner(bundle, lookupServiceAddress1); - owner2 = channel1.isOwner(bundle, lookupServiceAddress2); + owner1 = channel2.isOwner(bundle, brokerId1); + owner2 = channel1.isOwner(bundle, brokerId2); assertTrue(owner1); assertFalse(owner2); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigning, lookupServiceAddress1, 1)); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigning, brokerId1, 1)); assertFalse(channel1.isOwner(bundle)); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Owned, lookupServiceAddress1, 1)); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Owned, brokerId1, 1)); assertTrue(channel1.isOwner(bundle)); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, null, lookupServiceAddress1, 1)); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, null, brokerId1, 1)); assertFalse(channel1.isOwner(bundle)); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, null, lookupServiceAddress1, 1)); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, null, brokerId1, 1)); assertTrue(channel1.isOwner(bundle)); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Free, null, lookupServiceAddress1, 1)); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Free, null, brokerId1, 1)); assertFalse(channel1.isOwner(bundle)); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Deleted, null, lookupServiceAddress1, 1)); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Deleted, null, brokerId1, 1)); assertFalse(channel1.isOwner(bundle)); overrideTableView(channel1, bundle, null); assertFalse(channel1.isOwner(bundle)); } + @Test(priority = 15) + public void testGetOwnerAsync() throws Exception { + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Owned, brokerId1, 1)); + var owner = channel1.getOwnerAsync(bundle); + assertTrue(owner.isDone()); + assertEquals(brokerId1, owner.get().get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Owned, brokerId2, 1)); + owner = channel1.getOwnerAsync(bundle); + assertTrue(owner.isDone()); + assertEquals(brokerId2, owner.get().get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigning, brokerId1, 1)); + owner = channel1.getOwnerAsync(bundle); + assertTrue(!owner.isDone()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigning, brokerId2, 1)); + owner = channel1.getOwnerAsync(bundle); + assertTrue(owner.isDone()); + assertEquals(brokerId2, owner.get().get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, brokerId1, 1)); + owner = channel1.getOwnerAsync(bundle); + assertTrue(!owner.isDone()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, brokerId2, 1)); + owner = channel1.getOwnerAsync(bundle); + assertTrue(owner.isDone()); + assertEquals(brokerId2, owner.get().get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, null, brokerId1, 1)); + owner = channel1.getOwnerAsync(bundle); + assertTrue(owner.isDone()); + assertEquals(Optional.empty(), owner.get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, null, brokerId1, 1)); + owner = channel1.getOwnerAsync(bundle); + assertTrue(owner.isDone()); + assertEquals(brokerId1, owner.get().get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, null, brokerId2, 1)); + owner = channel1.getOwnerAsync(bundle); + assertTrue(owner.isDone()); + assertEquals(brokerId2, owner.get().get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Free, null, brokerId1, 1)); + owner = channel1.getOwnerAsync(bundle); + assertTrue(owner.isDone()); + assertEquals(Optional.empty(), owner.get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Deleted, null, brokerId1, 1)); + owner = channel1.getOwnerAsync(bundle); + assertTrue(owner.isDone()); + assertTrue(owner.isCompletedExceptionally()); + + overrideTableView(channel1, bundle, null); + owner = channel1.getOwnerAsync(bundle); + assertTrue(owner.isDone()); + assertEquals(Optional.empty(), owner.get()); + } + @Test(priority = 16) public void splitAndRetryFailureTest() throws Exception { - channel1.publishAssignEventAsync(bundle3, lookupServiceAddress1); - waitUntilNewOwner(channel1, bundle3, lookupServiceAddress1); - waitUntilNewOwner(channel2, bundle3, lookupServiceAddress1); + channel1.publishAssignEventAsync(bundle3, brokerId1); + waitUntilNewOwner(channel1, bundle3, brokerId1); + waitUntilNewOwner(channel2, bundle3, brokerId1); var ownerAddr1 = channel1.getOwnerAsync(bundle3).get(); var ownerAddr2 = channel2.getOwnerAsync(bundle3).get(); - assertEquals(ownerAddr1, Optional.of(lookupServiceAddress1)); - assertEquals(ownerAddr2, Optional.of(lookupServiceAddress1)); + assertEquals(ownerAddr1, Optional.of(brokerId1)); + assertEquals(ownerAddr2, Optional.of(brokerId1)); assertTrue(ownerAddr1.isPresent()); NamespaceService namespaceService = pulsar1.getNamespaceService(); @@ -1372,7 +1427,7 @@ public void splitAndRetryFailureTest() throws Exception { }); var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; ((ServiceUnitStateChannelImpl) leader) - .monitorOwnerships(List.of(lookupServiceAddress1, lookupServiceAddress2)); + .monitorOwnerships(List.of(brokerId1, brokerId2)); waitUntilState(leader, bundle3, Deleted); waitUntilState(channel1, bundle3, Deleted); waitUntilState(channel2, bundle3, Deleted); @@ -1383,14 +1438,14 @@ public void splitAndRetryFailureTest() throws Exception { validateEventCounters(channel1, 1, 0, 1, 0, 0, 0); validateEventCounters(channel2, 0, 0, 0, 0, 0, 0); - waitUntilNewOwner(channel1, childBundle31, lookupServiceAddress1); - waitUntilNewOwner(channel1, childBundle32, lookupServiceAddress1); - waitUntilNewOwner(channel2, childBundle31, lookupServiceAddress1); - waitUntilNewOwner(channel2, childBundle32, lookupServiceAddress1); - assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(childBundle31).get()); - assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(childBundle32).get()); - assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle31).get()); - assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle32).get()); + waitUntilNewOwner(channel1, childBundle31, brokerId1); + waitUntilNewOwner(channel1, childBundle32, brokerId1); + waitUntilNewOwner(channel2, childBundle31, brokerId1); + waitUntilNewOwner(channel2, childBundle32, brokerId1); + assertEquals(Optional.of(brokerId1), channel1.getOwnerAsync(childBundle31).get()); + assertEquals(Optional.of(brokerId1), channel1.getOwnerAsync(childBundle32).get()); + assertEquals(Optional.of(brokerId1), channel2.getOwnerAsync(childBundle31).get()); + assertEquals(Optional.of(brokerId1), channel2.getOwnerAsync(childBundle32).get()); // try the monitor and check the monitor moves `Deleted` -> `Init` @@ -1401,9 +1456,9 @@ public void splitAndRetryFailureTest() throws Exception { "stateTombstoneDelayTimeInMillis", 1, true); ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( - List.of(lookupServiceAddress1, lookupServiceAddress2)); + List.of(brokerId1, brokerId2)); ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( - List.of(lookupServiceAddress1, lookupServiceAddress2)); + List.of(brokerId1, brokerId2)); waitUntilState(channel1, bundle3, Init); waitUntilState(channel2, bundle3, Init); @@ -1439,12 +1494,12 @@ public void testOverrideInactiveBrokerStateData() String leader = channel1.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); String leader2 = channel2.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); assertEquals(leader, leader2); - if (leader.equals(lookupServiceAddress2)) { + if (leader.equals(brokerId2)) { leaderChannel = channel2; followerChannel = channel1; } - String broker = lookupServiceAddress1; + String broker = brokerId1; // test override states String releasingBundle = "public/releasing/0xfffffff0_0xffffffff"; @@ -1469,7 +1524,7 @@ public void testOverrideInactiveBrokerStateData() new ServiceUnitStateData(Owned, broker, null, 1)); // test stable metadata state - doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))) + doReturn(CompletableFuture.completedFuture(Optional.of(brokerId2))) .when(loadManager).selectAsync(any(), any()); leaderChannel.handleMetadataSessionEvent(SessionReestablished); followerChannel.handleMetadataSessionEvent(SessionReestablished); @@ -1480,11 +1535,11 @@ public void testOverrideInactiveBrokerStateData() leaderChannel.handleBrokerRegistrationEvent(broker, NotificationType.Deleted); followerChannel.handleBrokerRegistrationEvent(broker, NotificationType.Deleted); - waitUntilNewOwner(channel2, releasingBundle, lookupServiceAddress2); - waitUntilNewOwner(channel2, childBundle11, lookupServiceAddress2); - waitUntilNewOwner(channel2, childBundle12, lookupServiceAddress2); - waitUntilNewOwner(channel2, assigningBundle, lookupServiceAddress2); - waitUntilNewOwner(channel2, ownedBundle, lookupServiceAddress2); + waitUntilNewOwner(channel2, releasingBundle, brokerId2); + waitUntilNewOwner(channel2, childBundle11, brokerId2); + waitUntilNewOwner(channel2, childBundle12, brokerId2); + waitUntilNewOwner(channel2, assigningBundle, brokerId2); + waitUntilNewOwner(channel2, ownedBundle, brokerId2); assertEquals(Optional.empty(), channel2.getOwnerAsync(freeBundle).get()); assertTrue(channel2.getOwnerAsync(deletedBundle).isCompletedExceptionally()); assertTrue(channel2.getOwnerAsync(splittingBundle).isCompletedExceptionally()); @@ -1504,12 +1559,12 @@ public void testOverrideOrphanStateData() String leader = channel1.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); String leader2 = channel2.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); assertEquals(leader, leader2); - if (leader.equals(lookupServiceAddress2)) { + if (leader.equals(brokerId2)) { leaderChannel = channel2; followerChannel = channel1; } - String broker = lookupServiceAddress1; + String broker = brokerId1; // test override states String releasingBundle = "public/releasing/0xfffffff0_0xffffffff"; @@ -1534,19 +1589,19 @@ public void testOverrideOrphanStateData() new ServiceUnitStateData(Owned, broker, null, 1)); // test stable metadata state - doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))) + doReturn(CompletableFuture.completedFuture(Optional.of(brokerId2))) .when(loadManager).selectAsync(any(), any()); FieldUtils.writeDeclaredField(leaderChannel, "inFlightStateWaitingTimeInMillis", -1, true); FieldUtils.writeDeclaredField(followerChannel, "inFlightStateWaitingTimeInMillis", -1, true); ((ServiceUnitStateChannelImpl) leaderChannel) - .monitorOwnerships(List.of(lookupServiceAddress1, lookupServiceAddress2)); + .monitorOwnerships(List.of(brokerId1, brokerId2)); waitUntilNewOwner(channel2, releasingBundle, broker); waitUntilNewOwner(channel2, childBundle11, broker); waitUntilNewOwner(channel2, childBundle12, broker); - waitUntilNewOwner(channel2, assigningBundle, lookupServiceAddress2); + waitUntilNewOwner(channel2, assigningBundle, brokerId2); waitUntilNewOwner(channel2, ownedBundle, broker); assertEquals(Optional.empty(), channel2.getOwnerAsync(freeBundle).get()); assertTrue(channel2.getOwnerAsync(deletedBundle).isCompletedExceptionally()); @@ -1560,6 +1615,80 @@ public void testOverrideOrphanStateData() cleanTableViews(); } + @Test(priority = 19) + public void testActiveGetOwner() throws Exception { + + + // set the bundle owner is the broker + String broker = brokerId2; + String bundle = "public/owned/0xfffffff0_0xffffffff"; + overrideTableViews(bundle, + new ServiceUnitStateData(Owned, broker, null, 1)); + var owner = channel1.getOwnerAsync(bundle).get(5, TimeUnit.SECONDS).get(); + assertEquals(owner, broker); + + // simulate the owner is inactive + var spyRegistry = spy(new BrokerRegistryImpl(pulsar)); + doReturn(CompletableFuture.completedFuture(Optional.empty())) + .when(spyRegistry).lookupAsync(eq(broker)); + FieldUtils.writeDeclaredField(channel1, + "brokerRegistry", spyRegistry , true); + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 1000, true); + + + // verify getOwnerAsync times out because the owner is inactive now. + long start = System.currentTimeMillis(); + var ex = expectThrows(ExecutionException.class, () -> channel1.getOwnerAsync(bundle).get()); + assertTrue(ex.getCause() instanceof IllegalStateException); + assertTrue(System.currentTimeMillis() - start >= 1000); + + // simulate ownership cleanup(no selected owner) by the leader channel + doReturn(CompletableFuture.completedFuture(Optional.empty())) + .when(loadManager).selectAsync(any(), any()); + var leaderChannel = channel1; + String leader1 = channel1.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); + String leader2 = channel2.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); + assertEquals(leader1, leader2); + if (leader1.equals(brokerId2)) { + leaderChannel = channel2; + } + leaderChannel.handleMetadataSessionEvent(SessionReestablished); + FieldUtils.writeDeclaredField(leaderChannel, "lastMetadataSessionEventTimestamp", + System.currentTimeMillis() - (MAX_CLEAN_UP_DELAY_TIME_IN_SECS * 1000 + 1000), true); + leaderChannel.handleBrokerRegistrationEvent(broker, NotificationType.Deleted); + + // verify the ownership cleanup, and channel's getOwnerAsync returns empty result without timeout + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 20 * 1000, true); + start = System.currentTimeMillis(); + assertTrue(channel1.getOwnerAsync(bundle).get().isEmpty()); + assertTrue(System.currentTimeMillis() - start < 20_000); + + // simulate ownership cleanup(brokerId1 selected owner) by the leader channel + overrideTableViews(bundle, + new ServiceUnitStateData(Owned, broker, null, 1)); + doReturn(CompletableFuture.completedFuture(Optional.of(brokerId1))) + .when(loadManager).selectAsync(any(), any()); + leaderChannel.handleMetadataSessionEvent(SessionReestablished); + FieldUtils.writeDeclaredField(leaderChannel, "lastMetadataSessionEventTimestamp", + System.currentTimeMillis() - (MAX_CLEAN_UP_DELAY_TIME_IN_SECS * 1000 + 1000), true); + getCleanupJobs(leaderChannel).clear(); + leaderChannel.handleBrokerRegistrationEvent(broker, NotificationType.Deleted); + + // verify the ownership cleanup, and channel's getOwnerAsync returns brokerId1 without timeout + start = System.currentTimeMillis(); + assertEquals(brokerId1, channel1.getOwnerAsync(bundle).get().get()); + assertTrue(System.currentTimeMillis() - start < 20_000); + + // test clean-up + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); + FieldUtils.writeDeclaredField(channel1, + "brokerRegistry", registry , true); + cleanTableViews(); + + } private static ConcurrentHashMap>> getOwnerRequests( ServiceUnitStateChannel channel) throws IllegalAccessException { @@ -1671,7 +1800,7 @@ private void waitUntilStateWithMonitor(ServiceUnitStateChannel channel, String k .atMost(10, TimeUnit.SECONDS) .until(() -> { // wait until true ((ServiceUnitStateChannelImpl) channel) - .monitorOwnerships(List.of(lookupServiceAddress1, lookupServiceAddress2)); + .monitorOwnerships(List.of(brokerId1, brokerId2)); ServiceUnitStateData data = tv.get(key); ServiceUnitState actual = state(data); return actual == expected; @@ -1707,10 +1836,13 @@ private void overrideTableViews(String serviceUnit, ServiceUnitStateData val) th overrideTableView(channel2, serviceUnit, val); } - private static void overrideTableView(ServiceUnitStateChannel channel, String serviceUnit, ServiceUnitStateData val) + @Test(enabled = false) + public static void overrideTableView(ServiceUnitStateChannel channel, String serviceUnit, ServiceUnitStateData val) throws IllegalAccessException { var tv = (TableViewImpl) FieldUtils.readField(channel, "tableview", true); + var getOwnerRequests = (Map>) + FieldUtils.readField(channel, "getOwnerRequests", true); var cache = (ConcurrentMap) FieldUtils.readField(tv, "data", true); if(val == null){ @@ -1718,6 +1850,7 @@ private static void overrideTableView(ServiceUnitStateChannel channel, String se } else { cache.put(serviceUnit, val); } + getOwnerRequests.clear(); } private static void cleanOpsCounters(ServiceUnitStateChannel channel) @@ -1893,7 +2026,7 @@ ServiceUnitStateChannelImpl createChannel(PulsarService pulsar) var leaderElectionService = new LeaderElectionService( - pulsar.getCoordinationService(), pulsar.getSafeWebServiceAddress(), + pulsar.getCoordinationService(), pulsar.getBrokerId(), pulsar.getSafeWebServiceAddress(), state -> { if (state == LeaderElectionState.Leading) { channel.scheduleOwnershipMonitor(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java index f45e1405e1d21..87aaf4bac7fae 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java @@ -84,20 +84,20 @@ public void testFilterWithNamespaceIsolationPoliciesForPrimaryAndSecondaryBroker // a. available-brokers: broker1, broker2, broker3 => result: broker1 Map result = filter.filterAsync(new HashMap<>(Map.of( - "broker1", getLookupData(), - "broker2", getLookupData(), - "broker3", getLookupData())), namespaceName, getContext()).get(); - assertEquals(result.keySet(), Set.of("broker1")); + "broker1:8080", getLookupData(), + "broker2:8080", getLookupData(), + "broker3:8080", getLookupData())), namespaceName, getContext()).get(); + assertEquals(result.keySet(), Set.of("broker1:8080")); // b. available-brokers: broker2, broker3 => result: broker2 result = filter.filterAsync(new HashMap<>(Map.of( - "broker2", getLookupData(), - "broker3", getLookupData())), namespaceName, getContext()).get(); - assertEquals(result.keySet(), Set.of("broker2")); + "broker2:8080", getLookupData(), + "broker3:8080", getLookupData())), namespaceName, getContext()).get(); + assertEquals(result.keySet(), Set.of("broker2:8080")); // c. available-brokers: broker3 => result: NULL result = filter.filterAsync(new HashMap<>(Map.of( - "broker3", getLookupData())), namespaceName, getContext()).get(); + "broker3:8080", getLookupData())), namespaceName, getContext()).get(); assertTrue(result.isEmpty()); // 2. Namespace: primary=broker1, secondary=broker2, shared=broker3, min_limit = 2 @@ -105,20 +105,20 @@ public void testFilterWithNamespaceIsolationPoliciesForPrimaryAndSecondaryBroker // a. available-brokers: broker1, broker2, broker3 => result: broker1, broker2 result = filter.filterAsync(new HashMap<>(Map.of( - "broker1", getLookupData(), - "broker2", getLookupData(), - "broker3", getLookupData())), namespaceName, getContext()).get(); - assertEquals(result.keySet(), Set.of("broker1", "broker2")); + "broker1:8080", getLookupData(), + "broker2:8080", getLookupData(), + "broker3:8080", getLookupData())), namespaceName, getContext()).get(); + assertEquals(result.keySet(), Set.of("broker1:8080", "broker2:8080")); // b. available-brokers: broker2, broker3 => result: broker2 result = filter.filterAsync(new HashMap<>(Map.of( - "broker2", getLookupData(), - "broker3", getLookupData())), namespaceName, getContext()).get(); - assertEquals(result.keySet(), Set.of("broker2")); + "broker2:8080", getLookupData(), + "broker3:8080", getLookupData())), namespaceName, getContext()).get(); + assertEquals(result.keySet(), Set.of("broker2:8080")); // c. available-brokers: broker3 => result: NULL result = filter.filterAsync(new HashMap<>(Map.of( - "broker3", getLookupData())), namespaceName, getContext()).get(); + "broker3:8080", getLookupData())), namespaceName, getContext()).get(); assertTrue(result.isEmpty()); } @@ -142,31 +142,31 @@ public void testFilterWithPersistentOrNonPersistentDisabled() Map result = filter.filterAsync(new HashMap<>(Map.of( - "broker1", getLookupData(), - "broker2", getLookupData(), - "broker3", getLookupData())), namespaceBundle, getContext()).get(); - assertEquals(result.keySet(), Set.of("broker1", "broker2", "broker3")); + "broker1:8080", getLookupData(), + "broker2:8080", getLookupData(), + "broker3:8080", getLookupData())), namespaceBundle, getContext()).get(); + assertEquals(result.keySet(), Set.of("broker1:8080", "broker2:8080", "broker3:8080")); result = filter.filterAsync(new HashMap<>(Map.of( - "broker1", getLookupData(true, false), - "broker2", getLookupData(true, false), - "broker3", getLookupData())), namespaceBundle, getContext()).get(); - assertEquals(result.keySet(), Set.of("broker3")); + "broker1:8080", getLookupData(true, false), + "broker2:8080", getLookupData(true, false), + "broker3:8080", getLookupData())), namespaceBundle, getContext()).get(); + assertEquals(result.keySet(), Set.of("broker3:8080")); doReturn(false).when(namespaceBundle).hasNonPersistentTopic(); result = filter.filterAsync(new HashMap<>(Map.of( - "broker1", getLookupData(), - "broker2", getLookupData(), - "broker3", getLookupData())), namespaceBundle, getContext()).get(); - assertEquals(result.keySet(), Set.of("broker1", "broker2", "broker3")); + "broker1:8080", getLookupData(), + "broker2:8080", getLookupData(), + "broker3:8080", getLookupData())), namespaceBundle, getContext()).get(); + assertEquals(result.keySet(), Set.of("broker1:8080", "broker2:8080", "broker3:8080")); result = filter.filterAsync(new HashMap<>(Map.of( - "broker1", getLookupData(false, true), - "broker2", getLookupData(), - "broker3", getLookupData())), namespaceBundle, getContext()).get(); - assertEquals(result.keySet(), Set.of("broker2", "broker3")); + "broker1:8080", getLookupData(false, true), + "broker2:8080", getLookupData(), + "broker3:8080", getLookupData())), namespaceBundle, getContext()).get(); + assertEquals(result.keySet(), Set.of("broker2:8080", "broker3:8080")); } private void setIsolationPolicies(SimpleResourceAllocationPolicies policies, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java index 4eec612477758..0ff64616973d9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java @@ -100,6 +100,7 @@ import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; +import org.assertj.core.api.Assertions; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -168,18 +169,18 @@ public LoadManagerContext setupContext(){ var ctx = getContext(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000)); - topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 1000000, 3000000)); - topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("my-tenant/my-namespaceC", 2000000, 4000000)); - topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD", 2000000, 6000000)); - topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 2000000, 7000000)); + topBundlesLoadDataStore.pushAsync("broker1:8080", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000)); + topBundlesLoadDataStore.pushAsync("broker2:8080", getTopBundlesLoad("my-tenant/my-namespaceB", 1000000, 3000000)); + topBundlesLoadDataStore.pushAsync("broker3:8080", getTopBundlesLoad("my-tenant/my-namespaceC", 2000000, 4000000)); + topBundlesLoadDataStore.pushAsync("broker4:8080", getTopBundlesLoad("my-tenant/my-namespaceD", 2000000, 6000000)); + topBundlesLoadDataStore.pushAsync("broker5:8080", getTopBundlesLoad("my-tenant/my-namespaceE", 2000000, 7000000)); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 2, "broker1")); - brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 4, "broker2")); - brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx, 6, "broker3")); - brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 80, "broker4")); - brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 90, "broker5")); + brokerLoadDataStore.pushAsync("broker1:8080", getCpuLoad(ctx, 2, "broker1:8080")); + brokerLoadDataStore.pushAsync("broker2:8080", getCpuLoad(ctx, 4, "broker2:8080")); + brokerLoadDataStore.pushAsync("broker3:8080", getCpuLoad(ctx, 6, "broker3:8080")); + brokerLoadDataStore.pushAsync("broker4:8080", getCpuLoad(ctx, 80, "broker4:8080")); + brokerLoadDataStore.pushAsync("broker5:8080", getCpuLoad(ctx, 90, "broker5:8080")); return ctx; } @@ -192,9 +193,9 @@ public LoadManagerContext setupContext(int clusterSize) { Random rand = new Random(); for (int i = 0; i < clusterSize; i++) { int brokerLoad = rand.nextInt(1000); - brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx, brokerLoad, "broker" + i)); + brokerLoadDataStore.pushAsync("broker" + i + ":8080", getCpuLoad(ctx, brokerLoad, "broker" + i + ":8080")); int bundleLoad = rand.nextInt(brokerLoad + 1); - topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i, + topBundlesLoadDataStore.pushAsync("broker" + i + ":8080", getTopBundlesLoad("my-tenant/my-namespace" + i, bundleLoad, brokerLoad - bundleLoad)); } return ctx; @@ -209,14 +210,14 @@ public LoadManagerContext setupContextLoadSkewedOverload(int clusterSize) { int i = 0; for (; i < clusterSize-1; i++) { int brokerLoad = 1; - topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i, + topBundlesLoadDataStore.pushAsync("broker" + i + ":8080", getTopBundlesLoad("my-tenant/my-namespace" + i, 300_000, 700_000)); - brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx, brokerLoad, "broker" + i)); + brokerLoadDataStore.pushAsync("broker" + i + ":8080", getCpuLoad(ctx, brokerLoad, "broker" + i + ":8080")); } int brokerLoad = 100; - topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i, + topBundlesLoadDataStore.pushAsync("broker" + i + ":8080", getTopBundlesLoad("my-tenant/my-namespace" + i, 30_000_000, 70_000_000)); - brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx, brokerLoad, "broker" + i)); + brokerLoadDataStore.pushAsync("broker" + i + ":8080", getCpuLoad(ctx, brokerLoad, "broker" + i + ":8080")); return ctx; } @@ -230,21 +231,21 @@ public LoadManagerContext setupContextLoadSkewedUnderload(int clusterSize) { int i = 0; for (; i < clusterSize-2; i++) { int brokerLoad = 98; - topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i, + topBundlesLoadDataStore.pushAsync("broker" + i + ":8080", getTopBundlesLoad("my-tenant/my-namespace" + i, 30_000_000, 70_000_000)); - brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx, brokerLoad, "broker" + i)); + brokerLoadDataStore.pushAsync("broker" + i + ":8080", getCpuLoad(ctx, brokerLoad, "broker" + i + ":8080")); } int brokerLoad = 99; - topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i, + topBundlesLoadDataStore.pushAsync("broker" + i + ":8080", getTopBundlesLoad("my-tenant/my-namespace" + i, 30_000_000, 70_000_000)); - brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx, brokerLoad, "broker" + i)); + brokerLoadDataStore.pushAsync("broker" + i + ":8080", getCpuLoad(ctx, brokerLoad, "broker" + i + ":8080")); i++; brokerLoad = 1; - topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i, + topBundlesLoadDataStore.pushAsync("broker" + i + ":8080", getTopBundlesLoad("my-tenant/my-namespace" + i, 300_000, 700_000)); - brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx, brokerLoad, "broker" + i)); + brokerLoadDataStore.pushAsync("broker" + i + ":8080", getCpuLoad(ctx, brokerLoad, "broker" + i + ":8080")); return ctx; } @@ -474,11 +475,11 @@ public void startProducer() throws LoadDataStoreException { BrokerRegistry brokerRegistry = mock(BrokerRegistry.class); doReturn(CompletableFuture.completedFuture(Map.of( - "broker1", getMockBrokerLookupData(), - "broker2", getMockBrokerLookupData(), - "broker3", getMockBrokerLookupData(), - "broker4", getMockBrokerLookupData(), - "broker5", getMockBrokerLookupData() + "broker1:8080", getMockBrokerLookupData(), + "broker2:8080", getMockBrokerLookupData(), + "broker3:8080", getMockBrokerLookupData(), + "broker4:8080", getMockBrokerLookupData(), + "broker5:8080", getMockBrokerLookupData() ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync(); doReturn(conf).when(ctx).brokerConfiguration(); doReturn(brokerLoadDataStore).when(ctx).brokerLoadDataStore(); @@ -526,11 +527,11 @@ public void testEmptyTopBundlesLoadData() { var ctx = getContext(); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 2, "broker1")); - brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 4, "broker2")); - brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx, 6, "broker3")); - brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 80, "broker4")); - brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 90, "broker5")); + brokerLoadDataStore.pushAsync("broker1:8080", getCpuLoad(ctx, 2, "broker1:8080")); + brokerLoadDataStore.pushAsync("broker2:8080", getCpuLoad(ctx, 4, "broker2:8080")); + brokerLoadDataStore.pushAsync("broker3:8080", getCpuLoad(ctx, 6, "broker3:8080")); + brokerLoadDataStore.pushAsync("broker4:8080", getCpuLoad(ctx, 80, "broker4:8080")); + brokerLoadDataStore.pushAsync("broker5:8080", getCpuLoad(ctx, 90, "broker5:8080")); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); @@ -550,11 +551,11 @@ public void testOutDatedLoadData() throws IllegalAccessException { assertEquals(res.size(), 2); - FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker1").get(), "updatedAt", 0, true); - FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker2").get(), "updatedAt", 0, true); - FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker3").get(), "updatedAt", 0, true); - FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker4").get(), "updatedAt", 0, true); - FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker5").get(), "updatedAt", 0, true); + FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker1:8080").get(), "updatedAt", 0, true); + FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker2:8080").get(), "updatedAt", 0, true); + FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker3:8080").get(), "updatedAt", 0, true); + FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker4:8080").get(), "updatedAt", 0, true); + FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker5:8080").get(), "updatedAt", 0, true); res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); @@ -571,20 +572,20 @@ public void testRecentlyUnloadedBrokers() { Map recentlyUnloadedBrokers = new HashMap<>(); var oldTS = System.currentTimeMillis() - ctx.brokerConfiguration() .getLoadBalancerBrokerLoadDataTTLInSeconds() * 1001; - recentlyUnloadedBrokers.put("broker1", oldTS); + recentlyUnloadedBrokers.put("broker1:8080", oldTS); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), recentlyUnloadedBrokers); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + expected.add(new UnloadDecision(new Unload("broker5:8080", bundleE1, Optional.of("broker1:8080")), Success, Overloaded)); - expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + expected.add(new UnloadDecision(new Unload("broker4:8080", bundleD1, Optional.of("broker2:8080")), Success, Overloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), setupLoadAvg); assertEquals(counter.getLoadStd(), setupLoadStd); var now = System.currentTimeMillis(); - recentlyUnloadedBrokers.put("broker1", now); + recentlyUnloadedBrokers.put("broker1:8080", now); res = transferShedder.findBundlesForUnloading(ctx, Map.of(), recentlyUnloadedBrokers); assertTrue(res.isEmpty()); @@ -604,9 +605,9 @@ public void testRecentlyUnloadedBundles() { recentlyUnloadedBundles.put(bundleD2, now); var res = transferShedder.findBundlesForUnloading(ctx, recentlyUnloadedBundles, Map.of()); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker3", + expected.add(new UnloadDecision(new Unload("broker3:8080", "my-tenant/my-namespaceC/0x00000000_0x0FFFFFFF", - Optional.of("broker1")), + Optional.of("broker1:8080")), Success, Overloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), setupLoadAvg); @@ -639,13 +640,13 @@ public void testBundlesWithIsolationPolicies() { isolationPoliciesHelper, antiAffinityGroupPolicyHelper)); setIsolationPolicies(allocationPoliciesSpy, "my-tenant/my-namespaceE", - Set.of("broker5"), Set.of(), Set.of(), 1); + Set.of("broker5:8080"), Set.of(), Set.of(), 1); var ctx = setupContext(); ctx.brokerConfiguration().setLoadBalancerSheddingBundlesWithPoliciesEnabled(true); doReturn(ctx.brokerConfiguration()).when(pulsar).getConfiguration(); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker1")), + expected.add(new UnloadDecision(new Unload("broker4:8080", bundleD1, Optional.of("broker1:8080")), Success, Overloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), setupLoadAvg); @@ -656,7 +657,7 @@ public void testBundlesWithIsolationPolicies() { res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); expected = new HashSet<>(); - expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.empty()), + expected.add(new UnloadDecision(new Unload("broker4:8080", bundleD1, Optional.empty()), Success, Overloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), setupLoadAvg); @@ -777,7 +778,7 @@ public void testBundlesWithAntiAffinityGroup() throws MetadataStoreException { }).when(antiAffinityGroupPolicyHelper).filterAsync(any(), any()); var res2 = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected2 = new HashSet<>(); - expected2.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + expected2.add(new UnloadDecision(new Unload("broker5:8080", bundleE1, Optional.of("broker1:8080")), Success, Overloaded)); assertEquals(res2, expected2); assertEquals(counter.getLoadAvg(), setupLoadAvg); @@ -853,22 +854,22 @@ public void testTargetStd() { var ctx = getContext(); BrokerRegistry brokerRegistry = mock(BrokerRegistry.class); doReturn(CompletableFuture.completedFuture(Map.of( - "broker1", mock(BrokerLookupData.class), - "broker2", mock(BrokerLookupData.class), - "broker3", mock(BrokerLookupData.class) + "broker1:8080", mock(BrokerLookupData.class), + "broker2:8080", mock(BrokerLookupData.class), + "broker3:8080", mock(BrokerLookupData.class) ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync(); doReturn(brokerRegistry).when(ctx).brokerRegistry(); ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 10, "broker1")); - brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 20, "broker2")); - brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx, 30, "broker3")); + brokerLoadDataStore.pushAsync("broker1:8080", getCpuLoad(ctx, 10, "broker1:8080")); + brokerLoadDataStore.pushAsync("broker2:8080", getCpuLoad(ctx, 20, "broker2:8080")); + brokerLoadDataStore.pushAsync("broker3:8080", getCpuLoad(ctx, 30, "broker3:8080")); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 30, 30)); - topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 40, 40)); - topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("my-tenant/my-namespaceC", 50, 50)); + topBundlesLoadDataStore.pushAsync("broker1:8080", getTopBundlesLoad("my-tenant/my-namespaceA", 30, 30)); + topBundlesLoadDataStore.pushAsync("broker2:8080", getTopBundlesLoad("my-tenant/my-namespaceB", 40, 40)); + topBundlesLoadDataStore.pushAsync("broker3:8080", getTopBundlesLoad("my-tenant/my-namespaceC", 50, 50)); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); @@ -884,11 +885,11 @@ public void testSingleTopBundlesLoadData() { TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContext(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1)); - topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 2)); - topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("my-tenant/my-namespaceC", 6)); - topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD", 10)); - topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 70)); + topBundlesLoadDataStore.pushAsync("broker1:8080", getTopBundlesLoad("my-tenant/my-namespaceA", 1)); + topBundlesLoadDataStore.pushAsync("broker2:8080", getTopBundlesLoad("my-tenant/my-namespaceB", 2)); + topBundlesLoadDataStore.pushAsync("broker3:8080", getTopBundlesLoad("my-tenant/my-namespaceC", 6)); + topBundlesLoadDataStore.pushAsync("broker4:8080", getTopBundlesLoad("my-tenant/my-namespaceD", 10)); + topBundlesLoadDataStore.pushAsync("broker5:8080", getTopBundlesLoad("my-tenant/my-namespaceE", 70)); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); assertTrue(res.isEmpty()); @@ -903,14 +904,14 @@ public void testBundleThroughputLargerThanOffloadThreshold() { TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContext(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD", 1000000000, 1000000000)); - topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 1000000000, 1000000000)); + topBundlesLoadDataStore.pushAsync("broker4:8080", getTopBundlesLoad("my-tenant/my-namespaceD", 1000000000, 1000000000)); + topBundlesLoadDataStore.pushAsync("broker5:8080", getTopBundlesLoad("my-tenant/my-namespaceE", 1000000000, 1000000000)); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker3", + expected.add(new UnloadDecision(new Unload("broker3:8080", "my-tenant/my-namespaceC/0x00000000_0x0FFFFFFF", - Optional.of("broker1")), + Optional.of("broker1:8080")), Success, Overloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), setupLoadAvg); @@ -924,12 +925,12 @@ public void testTargetStdAfterTransfer() { TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContext(); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 55, "broker4")); - brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 65, "broker5")); + brokerLoadDataStore.pushAsync("broker4:8080", getCpuLoad(ctx, 55, "broker4:8080")); + brokerLoadDataStore.pushAsync("broker5:8080", getCpuLoad(ctx, 65, "broker5:8080")); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + expected.add(new UnloadDecision(new Unload("broker5:8080", bundleE1, Optional.of("broker1:8080")), Success, Overloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), 0.26400000000000007); @@ -945,43 +946,43 @@ public void testUnloadBundlesGreaterThanTargetThroughput() throws IllegalAccessE var brokerRegistry = mock(BrokerRegistry.class); doReturn(brokerRegistry).when(ctx).brokerRegistry(); doReturn(CompletableFuture.completedFuture(Map.of( - "broker1", mock(BrokerLookupData.class), - "broker2", mock(BrokerLookupData.class) + "broker1:8080", mock(BrokerLookupData.class), + "broker2:8080", mock(BrokerLookupData.class) ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000, 3000000, 4000000, 5000000)); - topBundlesLoadDataStore.pushAsync("broker2", + topBundlesLoadDataStore.pushAsync("broker1:8080", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000, 3000000, 4000000, 5000000)); + topBundlesLoadDataStore.pushAsync("broker2:8080", getTopBundlesLoad("my-tenant/my-namespaceB", 100000000, 180000000, 220000000, 250000000, 250000000)); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 10, "broker1")); - brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 1000, "broker2")); + brokerLoadDataStore.pushAsync("broker1:8080", getCpuLoad(ctx, 10, "broker1:8080")); + brokerLoadDataStore.pushAsync("broker2:8080", getCpuLoad(ctx, 1000, "broker2:8080")); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); expected.add(new UnloadDecision( - new Unload("broker2", "my-tenant/my-namespaceB/0x00000000_0x1FFFFFFF", Optional.of("broker1")), + new Unload("broker2:8080", "my-tenant/my-namespaceB/0x00000000_0x1FFFFFFF", Optional.of("broker1:8080")), Success, Overloaded)); expected.add(new UnloadDecision( - new Unload("broker2", "my-tenant/my-namespaceB/0x1FFFFFFF_0x2FFFFFFF", Optional.of("broker1")), + new Unload("broker2:8080", "my-tenant/my-namespaceB/0x1FFFFFFF_0x2FFFFFFF", Optional.of("broker1:8080")), Success, Overloaded)); expected.add(new UnloadDecision( - new Unload("broker2", "my-tenant/my-namespaceB/0x2FFFFFFF_0x3FFFFFFF", Optional.of("broker1")), + new Unload("broker2:8080", "my-tenant/my-namespaceB/0x2FFFFFFF_0x3FFFFFFF", Optional.of("broker1:8080")), Success, Overloaded)); expected.add(new UnloadDecision( - new Unload("broker1", "my-tenant/my-namespaceA/0x00000000_0x1FFFFFFF", Optional.of("broker2")), + new Unload("broker1:8080", "my-tenant/my-namespaceA/0x00000000_0x1FFFFFFF", Optional.of("broker2:8080")), Success, Overloaded)); expected.add(new UnloadDecision( - new Unload("broker1", "my-tenant/my-namespaceA/0x1FFFFFFF_0x2FFFFFFF", Optional.of("broker2")), + new Unload("broker1:8080", "my-tenant/my-namespaceA/0x1FFFFFFF_0x2FFFFFFF", Optional.of("broker2:8080")), Success, Overloaded)); expected.add(new UnloadDecision( - new Unload("broker1","my-tenant/my-namespaceA/0x2FFFFFFF_0x3FFFFFFF", Optional.of("broker2")), + new Unload("broker1:8080","my-tenant/my-namespaceA/0x2FFFFFFF_0x3FFFFFFF", Optional.of("broker2:8080")), Success, Overloaded)); expected.add(new UnloadDecision( - new Unload("broker1","my-tenant/my-namespaceA/0x3FFFFFFF_0x4FFFFFFF", Optional.of("broker2")), + new Unload("broker1:8080","my-tenant/my-namespaceA/0x3FFFFFFF_0x4FFFFFFF", Optional.of("broker2:8080")), Success, Overloaded)); assertEquals(counter.getLoadAvg(), 5.05); assertEquals(counter.getLoadStd(), 4.95); @@ -1001,20 +1002,20 @@ public void testSkipBundlesGreaterThanTargetThroughputAfterSplit() { var brokerRegistry = mock(BrokerRegistry.class); doReturn(brokerRegistry).when(ctx).brokerRegistry(); doReturn(CompletableFuture.completedFuture(Map.of( - "broker1", mock(BrokerLookupData.class), - "broker2", mock(BrokerLookupData.class) + "broker1:8080", mock(BrokerLookupData.class), + "broker2:8080", mock(BrokerLookupData.class) ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker1", + topBundlesLoadDataStore.pushAsync("broker1:8080", getTopBundlesLoad("my-tenant/my-namespaceA", 1, 500000000)); - topBundlesLoadDataStore.pushAsync("broker2", + topBundlesLoadDataStore.pushAsync("broker2:8080", getTopBundlesLoad("my-tenant/my-namespaceB", 500000000, 500000000)); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 50, "broker1")); - brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 100, "broker2")); + brokerLoadDataStore.pushAsync("broker1:8080", getCpuLoad(ctx, 50, "broker1:8080")); + brokerLoadDataStore.pushAsync("broker2:8080", getCpuLoad(ctx, 100, "broker2:8080")); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); @@ -1032,24 +1033,24 @@ public void testUnloadBundlesLessThanTargetThroughputAfterSplit() throws Illegal var brokerRegistry = mock(BrokerRegistry.class); doReturn(brokerRegistry).when(ctx).brokerRegistry(); doReturn(CompletableFuture.completedFuture(Map.of( - "broker1", mock(BrokerLookupData.class), - "broker2", mock(BrokerLookupData.class) + "broker1:8080", mock(BrokerLookupData.class), + "broker2:8080", mock(BrokerLookupData.class) ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000, 3000000, 4000000, 5000000)); - topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 490000000, 510000000)); + topBundlesLoadDataStore.pushAsync("broker1:8080", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000, 3000000, 4000000, 5000000)); + topBundlesLoadDataStore.pushAsync("broker2:8080", getTopBundlesLoad("my-tenant/my-namespaceB", 490000000, 510000000)); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 10, "broker1")); - brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 1000, "broker2")); + brokerLoadDataStore.pushAsync("broker1:8080", getCpuLoad(ctx, 10, "broker1:8080")); + brokerLoadDataStore.pushAsync("broker2:8080", getCpuLoad(ctx, 1000, "broker2:8080")); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); expected.add(new UnloadDecision( - new Unload("broker2", "my-tenant/my-namespaceB/0x00000000_0x0FFFFFFF", Optional.of("broker1")), + new Unload("broker2:8080", "my-tenant/my-namespaceB/0x00000000_0x0FFFFFFF", Optional.of("broker1:8080")), Success, Overloaded)); assertEquals(counter.getLoadAvg(), 5.05); assertEquals(counter.getLoadStd(), 4.95); @@ -1070,30 +1071,30 @@ public void testUnloadBundlesGreaterThanTargetThroughputAfterSplit() throws Ille var brokerRegistry = mock(BrokerRegistry.class); doReturn(brokerRegistry).when(ctx).brokerRegistry(); doReturn(CompletableFuture.completedFuture(Map.of( - "broker1", mock(BrokerLookupData.class), - "broker2", mock(BrokerLookupData.class) + "broker1:8080", mock(BrokerLookupData.class), + "broker2:8080", mock(BrokerLookupData.class) ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 2400000, 2400000)); - topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 5000000, 5000000)); + topBundlesLoadDataStore.pushAsync("broker1:8080", getTopBundlesLoad("my-tenant/my-namespaceA", 2400000, 2400000)); + topBundlesLoadDataStore.pushAsync("broker2:8080", getTopBundlesLoad("my-tenant/my-namespaceB", 5000000, 5000000)); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 48, "broker1")); - brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 100, "broker2")); + brokerLoadDataStore.pushAsync("broker1:8080", getCpuLoad(ctx, 48, "broker1:8080")); + brokerLoadDataStore.pushAsync("broker2:8080", getCpuLoad(ctx, 100, "broker2:8080")); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); expected.add(new UnloadDecision( - new Unload("broker1", - res.stream().filter(x -> x.getUnload().sourceBroker().equals("broker1")).findFirst().get() - .getUnload().serviceUnit(), Optional.of("broker2")), + new Unload("broker1:8080", + res.stream().filter(x -> x.getUnload().sourceBroker().equals("broker1:8080")).findFirst().get() + .getUnload().serviceUnit(), Optional.of("broker2:8080")), Success, Overloaded)); expected.add(new UnloadDecision( - new Unload("broker2", - res.stream().filter(x -> x.getUnload().sourceBroker().equals("broker2")).findFirst().get() - .getUnload().serviceUnit(), Optional.of("broker1")), + new Unload("broker2:8080", + res.stream().filter(x -> x.getUnload().sourceBroker().equals("broker2:8080")).findFirst().get() + .getUnload().serviceUnit(), Optional.of("broker1:8080")), Success, Overloaded)); assertEquals(counter.getLoadAvg(), 0.74); assertEquals(counter.getLoadStd(), 0.26); @@ -1111,18 +1112,18 @@ public void testMinBrokerWithZeroTraffic() throws IllegalAccessException { var ctx = setupContext(); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - var load = getCpuLoad(ctx, 4, "broker2"); + var load = getCpuLoad(ctx, 4, "broker2:8080"); FieldUtils.writeDeclaredField(load,"msgThroughputEMA", 0, true); - brokerLoadDataStore.pushAsync("broker2", load); - brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 55, "broker4")); - brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 65, "broker5")); + brokerLoadDataStore.pushAsync("broker2:8080", load); + brokerLoadDataStore.pushAsync("broker4:8080", getCpuLoad(ctx, 55, "broker4:8080")); + brokerLoadDataStore.pushAsync("broker5:8080", getCpuLoad(ctx, 65, "broker5:8080")); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + expected.add(new UnloadDecision(new Unload("broker5:8080", bundleE1, Optional.of("broker1:8080")), Success, Overloaded)); - expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + expected.add(new UnloadDecision(new Unload("broker4:8080", bundleD1, Optional.of("broker2:8080")), Success, Underloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), 0.26400000000000007); @@ -1136,17 +1137,17 @@ public void testMinBrokerWithLowerLoadThanAvg() throws IllegalAccessException { var ctx = setupContext(); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - var load = getCpuLoad(ctx, 3 , "broker2"); - brokerLoadDataStore.pushAsync("broker2", load); - brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 55, "broker4")); - brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 65, "broker5")); + var load = getCpuLoad(ctx, 3 , "broker2:8080"); + brokerLoadDataStore.pushAsync("broker2:8080", load); + brokerLoadDataStore.pushAsync("broker4:8080", getCpuLoad(ctx, 55, "broker4:8080")); + brokerLoadDataStore.pushAsync("broker5:8080", getCpuLoad(ctx, 65, "broker5:8080")); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + expected.add(new UnloadDecision(new Unload("broker5:8080", bundleE1, Optional.of("broker1:8080")), Success, Overloaded)); - expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + expected.add(new UnloadDecision(new Unload("broker4:8080", bundleD1, Optional.of("broker2:8080")), Success, Underloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), 0.262); @@ -1162,9 +1163,9 @@ public void testMaxNumberOfTransfersPerShedderCycle() { .setLoadBalancerMaxNumberOfBrokerSheddingPerCycle(10); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + expected.add(new UnloadDecision(new Unload("broker5:8080", bundleE1, Optional.of("broker1:8080")), Success, Overloaded)); - expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + expected.add(new UnloadDecision(new Unload("broker4:8080", bundleD1, Optional.of("broker2:8080")), Success, Overloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), setupLoadAvg); @@ -1188,9 +1189,9 @@ public void testLoadBalancerSheddingConditionHitCountThreshold() { } var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + expected.add(new UnloadDecision(new Unload("broker5:8080", bundleE1, Optional.of("broker1:8080")), Success, Overloaded)); - expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + expected.add(new UnloadDecision(new Unload("broker4:8080", bundleD1, Optional.of("broker2:8080")), Success, Overloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), setupLoadAvg); @@ -1203,13 +1204,13 @@ public void testRemainingTopBundles() { TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContext(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 2000000, 3000000)); + topBundlesLoadDataStore.pushAsync("broker5:8080", getTopBundlesLoad("my-tenant/my-namespaceE", 2000000, 3000000)); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + expected.add(new UnloadDecision(new Unload("broker5:8080", bundleE1, Optional.of("broker1:8080")), Success, Overloaded)); - expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + expected.add(new UnloadDecision(new Unload("broker4:8080", bundleD1, Optional.of("broker2:8080")), Success, Overloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), setupLoadAvg); @@ -1223,14 +1224,14 @@ public void testLoadMoreThan100() throws IllegalAccessException { var ctx = setupContext(); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 200, "broker4")); - brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 1000, "broker5")); + brokerLoadDataStore.pushAsync("broker4:8080", getCpuLoad(ctx, 200, "broker4:8080")); + brokerLoadDataStore.pushAsync("broker5:8080", getCpuLoad(ctx, 1000, "broker5:8080")); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + expected.add(new UnloadDecision(new Unload("broker5:8080", bundleE1, Optional.of("broker1:8080")), Success, Overloaded)); - expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + expected.add(new UnloadDecision(new Unload("broker4:8080", bundleD1, Optional.of("broker2:8080")), Success, Overloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), 2.4240000000000004); @@ -1264,13 +1265,16 @@ public void testOverloadOutlier() { TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContextLoadSkewedOverload(100); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); - var expected = new HashSet(); - expected.add(new UnloadDecision( - new Unload("broker99", "my-tenant/my-namespace99/0x00000000_0x0FFFFFFF", - Optional.of("broker52")), Success, Overloaded)); - assertEquals(res, expected); - assertEquals(counter.getLoadAvg(), 0.019900000000000008); - assertEquals(counter.getLoadStd(), 0.09850375627355534); + Assertions.assertThat(res).isIn( + Set.of(new UnloadDecision( + new Unload("broker99:8080", "my-tenant/my-namespace99/0x00000000_0x0FFFFFFF", + Optional.of("broker52:8080")), Success, Overloaded)), + Set.of(new UnloadDecision( + new Unload("broker99:8080", "my-tenant/my-namespace99/0x00000000_0x0FFFFFFF", + Optional.of("broker83:8080")), Success, Overloaded)) + ); + assertEquals(counter.getLoadAvg(), 0.019900000000000008, 0.00001); + assertEquals(counter.getLoadStd(), 0.09850375627355534, 0.00001); } @Test @@ -1281,11 +1285,11 @@ public void testUnderloadOutlier() { var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); expected.add(new UnloadDecision( - new Unload("broker98", "my-tenant/my-namespace98/0x00000000_0x0FFFFFFF", - Optional.of("broker99")), Success, Underloaded)); + new Unload("broker98:8080", "my-tenant/my-namespace98/0x00000000_0x0FFFFFFF", + Optional.of("broker99:8080")), Success, Underloaded)); assertEquals(res, expected); - assertEquals(counter.getLoadAvg(), 0.9704000000000005); - assertEquals(counter.getLoadStd(), 0.09652895938523735); + assertEquals(counter.getLoadAvg(), 0.9704000000000005, 0.00001); + assertEquals(counter.getLoadStd(), 0.09652895938523735, 0.00001); } @Test @@ -1301,13 +1305,13 @@ public void testRandomLoadStats() { double[] loads = new double[numBrokers]; final Map availableBrokers = new HashMap<>(); for (int i = 0; i < loads.length; i++) { - availableBrokers.put("broker" + i, mock(BrokerLookupData.class)); + availableBrokers.put("broker" + i + ":8080", mock(BrokerLookupData.class)); } stats.update(loadStore, availableBrokers, Map.of(), conf); var brokerLoadDataStore = ctx.brokerLoadDataStore(); for (int i = 0; i < loads.length; i++) { - loads[i] = loadStore.get("broker" + i).get().getWeightedMaxEMA(); + loads[i] = loadStore.get("broker" + i + ":8080").get().getWeightedMaxEMA(); } int i = 0; int j = loads.length - 1; @@ -1342,8 +1346,8 @@ public void testHighVarianceLoadStats() { var conf = ctx.brokerConfiguration(); final Map availableBrokers = new HashMap<>(); for (int i = 0; i < loads.length; i++) { - availableBrokers.put("broker" + i, mock(BrokerLookupData.class)); - loadStore.pushAsync("broker" + i, getCpuLoad(ctx, loads[i], "broker" + i)); + availableBrokers.put("broker" + i + ":8080", mock(BrokerLookupData.class)); + loadStore.pushAsync("broker" + i + ":8080", getCpuLoad(ctx, loads[i], "broker" + i + ":8080")); } stats.update(loadStore, availableBrokers, Map.of(), conf); @@ -1361,8 +1365,8 @@ public void testLowVarianceLoadStats() { var conf = ctx.brokerConfiguration(); final Map availableBrokers = new HashMap<>(); for (int i = 0; i < loads.length; i++) { - availableBrokers.put("broker" + i, mock(BrokerLookupData.class)); - loadStore.pushAsync("broker" + i, getCpuLoad(ctx, loads[i], "broker" + i)); + availableBrokers.put("broker" + i + ":8080", mock(BrokerLookupData.class)); + loadStore.pushAsync("broker" + i + ":8080", getCpuLoad(ctx, loads[i], "broker" + i + ":8080")); } stats.update(loadStore, availableBrokers, Map.of(), conf); assertEquals(stats.avg(), 3.9449999999999994); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java index f486370400c92..d25cba2bd1bdd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java @@ -20,8 +20,6 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.fail; import static org.testng.AssertJUnit.assertTrue; import com.google.common.collect.Sets; @@ -29,6 +27,7 @@ import lombok.Cleanup; import lombok.Data; import lombok.NoArgsConstructor; +import org.apache.commons.lang.reflect.FieldUtils; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; @@ -40,7 +39,6 @@ import java.util.HashMap; import java.util.Map; import java.util.UUID; -import java.util.concurrent.ExecutionException; @Test(groups = "broker") public class LoadDataStoreTest extends MockedPulsarServiceBaseTest { @@ -76,7 +74,7 @@ public void testPushGetAndRemove() throws Exception { @Cleanup LoadDataStore loadDataStore = - LoadDataStoreFactory.create(pulsar.getClient(), topic, MyClass.class); + LoadDataStoreFactory.create(pulsar, topic, MyClass.class); loadDataStore.startProducer(); loadDataStore.startTableView(); MyClass myClass1 = new MyClass("1", 1); @@ -110,7 +108,7 @@ public void testForEach() throws Exception { @Cleanup LoadDataStore loadDataStore = - LoadDataStoreFactory.create(pulsar.getClient(), topic, Integer.class); + LoadDataStoreFactory.create(pulsar, topic, Integer.class); loadDataStore.startProducer(); loadDataStore.startTableView(); @@ -135,7 +133,7 @@ public void testForEach() throws Exception { public void testTableViewRestart() throws Exception { String topic = TopicDomain.persistent + "://" + NamespaceName.SYSTEM_NAMESPACE + "/" + UUID.randomUUID(); LoadDataStore loadDataStore = - LoadDataStoreFactory.create(pulsar.getClient(), topic, Integer.class); + LoadDataStoreFactory.create(pulsar, topic, Integer.class); loadDataStore.startProducer(); loadDataStore.startTableView(); @@ -145,43 +143,26 @@ public void testTableViewRestart() throws Exception { loadDataStore.closeTableView(); loadDataStore.pushAsync("1", 2).get(); - Exception ex = null; - try { - loadDataStore.get("1"); - } catch (IllegalStateException e) { - ex = e; - } - assertNotNull(ex); - loadDataStore.startTableView(); Awaitility.await().untilAsserted(() -> assertEquals(loadDataStore.get("1").get(), 2)); + + loadDataStore.pushAsync("1", 3).get(); + FieldUtils.writeField(loadDataStore, "tableViewLastUpdateTimestamp", 0 , true); + Awaitility.await().untilAsserted(() -> assertEquals(loadDataStore.get("1").get(), 3)); } @Test public void testProducerStop() throws Exception { String topic = TopicDomain.persistent + "://" + NamespaceName.SYSTEM_NAMESPACE + "/" + UUID.randomUUID(); LoadDataStore loadDataStore = - LoadDataStoreFactory.create(pulsar.getClient(), topic, Integer.class); + LoadDataStoreFactory.create(pulsar, topic, Integer.class); loadDataStore.startProducer(); loadDataStore.pushAsync("1", 1).get(); loadDataStore.removeAsync("1").get(); loadDataStore.close(); - try { - loadDataStore.pushAsync("2", 2).get(); - fail(); - } catch (ExecutionException ex) { - assertTrue(ex.getCause() instanceof IllegalStateException); - } - try { - loadDataStore.removeAsync("2").get(); - fail(); - } catch (ExecutionException ex) { - assertTrue(ex.getCause() instanceof IllegalStateException); - } - loadDataStore.startProducer(); - loadDataStore.pushAsync("3", 3).get(); - loadDataStore.removeAsync("3").get(); + loadDataStore.pushAsync("2", 2).get(); + loadDataStore.removeAsync("2").get(); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java index 5937af68ec7f2..824291c52da77 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java @@ -23,7 +23,6 @@ import static org.apache.pulsar.broker.resources.LoadBalanceResources.BUNDLE_DATA_BASE_PATH; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -31,6 +30,7 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.BoundType; import com.google.common.collect.Range; @@ -63,6 +63,7 @@ import org.apache.pulsar.broker.loadbalance.LoadBalancerTestingUtils; import org.apache.pulsar.broker.loadbalance.LoadData; import org.apache.pulsar.broker.loadbalance.LoadManager; +import org.apache.pulsar.broker.loadbalance.ResourceUnit; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared.BrokerTopicLoadingPredicate; import org.apache.pulsar.client.admin.Namespaces; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -83,16 +84,17 @@ import org.apache.pulsar.common.policies.data.ResourceQuota; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.apache.pulsar.common.util.PortManager; import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.Notification; import org.apache.pulsar.metadata.api.NotificationType; import org.apache.pulsar.metadata.api.extended.CreateOption; +import org.apache.pulsar.policies.data.loadbalancer.BrokerData; +import org.apache.pulsar.policies.data.loadbalancer.BundleData; import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; -import org.apache.pulsar.policies.data.loadbalancer.BrokerData; -import org.apache.pulsar.policies.data.loadbalancer.BundleData; import org.apache.pulsar.policies.data.loadbalancer.TimeAverageBrokerData; import org.apache.pulsar.policies.data.loadbalancer.TimeAverageMessageData; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; @@ -119,8 +121,9 @@ public class ModularLoadManagerImplTest { private PulsarService pulsar3; - private String primaryHost; - private String secondaryHost; + private String primaryBrokerId; + + private String secondaryBrokerId; private NamespaceBundleFactory nsFactory; @@ -167,16 +170,18 @@ void setup() throws Exception { config1.setLoadBalancerLoadSheddingStrategy("org.apache.pulsar.broker.loadbalance.impl.OverloadShedder"); config1.setClusterName("use"); config1.setWebServicePort(Optional.of(0)); + config1.setWebServicePortTls(Optional.of(0)); config1.setMetadataStoreUrl("zk:127.0.0.1:" + bkEnsemble.getZookeeperPort()); config1.setAdvertisedAddress("localhost"); config1.setBrokerShutdownTimeoutMs(0L); config1.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); config1.setBrokerServicePort(Optional.of(0)); + config1.setBrokerServicePortTls(Optional.of(0)); pulsar1 = new PulsarService(config1); pulsar1.start(); - primaryHost = String.format("%s:%d", "localhost", pulsar1.getListenPortHTTP().get()); + primaryBrokerId = pulsar1.getBrokerId(); url1 = new URL(pulsar1.getWebServiceAddress()); admin1 = PulsarAdmin.builder().serviceHttpUrl(url1.toString()).build(); @@ -186,11 +191,13 @@ void setup() throws Exception { config2.setLoadBalancerLoadSheddingStrategy("org.apache.pulsar.broker.loadbalance.impl.OverloadShedder"); config2.setClusterName("use"); config2.setWebServicePort(Optional.of(0)); + config2.setWebServicePortTls(Optional.of(0)); config2.setMetadataStoreUrl("zk:127.0.0.1:" + bkEnsemble.getZookeeperPort()); config2.setAdvertisedAddress("localhost"); config2.setBrokerShutdownTimeoutMs(0L); config2.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); config2.setBrokerServicePort(Optional.of(0)); + config2.setBrokerServicePortTls(Optional.of(0)); pulsar2 = new PulsarService(config2); pulsar2.start(); @@ -199,14 +206,16 @@ void setup() throws Exception { config.setLoadBalancerLoadSheddingStrategy("org.apache.pulsar.broker.loadbalance.impl.OverloadShedder"); config.setClusterName("use"); config.setWebServicePort(Optional.of(0)); + config.setWebServicePortTls(Optional.of(0)); config.setMetadataStoreUrl("zk:127.0.0.1:" + bkEnsemble.getZookeeperPort()); config.setAdvertisedAddress("localhost"); config.setBrokerShutdownTimeoutMs(0L); config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); config.setBrokerServicePort(Optional.of(0)); + config.setBrokerServicePortTls(Optional.of(0)); pulsar3 = new PulsarService(config); - secondaryHost = String.format("%s:%d", "localhost", pulsar2.getListenPortHTTP().get()); + secondaryBrokerId = pulsar2.getBrokerId(); url2 = new URL(pulsar2.getWebServiceAddress()); admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build(); @@ -226,7 +235,7 @@ void shutdown() throws Exception { pulsar2.close(); pulsar1.close(); - + if (pulsar3.isRunning()) { pulsar3.close(); } @@ -257,7 +266,7 @@ public void testCandidateConsistency() throws Exception { for (int i = 0; i < 2; ++i) { final ServiceUnitId serviceUnit = makeBundle(Integer.toString(i)); final String broker = primaryLoadManager.selectBrokerForAssignment(serviceUnit).get(); - if (broker.equals(primaryHost)) { + if (broker.equals(primaryBrokerId)) { foundFirst = true; } else { foundSecond = true; @@ -273,12 +282,12 @@ public void testCandidateConsistency() throws Exception { LoadData loadData = (LoadData) getField(primaryLoadManager, "loadData"); // Make sure the second broker is not in the internal map. - Awaitility.await().untilAsserted(() -> assertFalse(loadData.getBrokerData().containsKey(secondaryHost))); + Awaitility.await().untilAsserted(() -> assertFalse(loadData.getBrokerData().containsKey(secondaryBrokerId))); // Try 5 more selections, ensure they all go to the first broker. for (int i = 2; i < 7; ++i) { final ServiceUnitId serviceUnit = makeBundle(Integer.toString(i)); - assertEquals(primaryLoadManager.selectBrokerForAssignment(serviceUnit), primaryHost); + assertEquals(primaryLoadManager.selectBrokerForAssignment(serviceUnit), primaryBrokerId); } } @@ -300,7 +309,7 @@ public void testEvenBundleDistribution() throws Exception { // one bundle. pulsar1.getLocalMetadataStore().getMetadataCache(BundleData.class).create(firstBundleDataPath, bundleData).join(); for (final NamespaceBundle bundle : bundles) { - if (primaryLoadManager.selectBrokerForAssignment(bundle).equals(primaryHost)) { + if (primaryLoadManager.selectBrokerForAssignment(bundle).equals(primaryBrokerId)) { ++numAssignedToPrimary; } else { ++numAssignedToSecondary; @@ -314,52 +323,52 @@ public void testEvenBundleDistribution() throws Exception { } - + @Test public void testBrokerAffinity() throws Exception { // Start broker 3 pulsar3.start(); - + final String tenant = "test"; final String cluster = "test"; String namespace = tenant + "/" + cluster + "/" + "test"; String topic = "persistent://" + namespace + "/my-topic1"; - admin1.clusters().createCluster(cluster, ClusterData.builder().serviceUrl("http://" + pulsar1.getAdvertisedAddress()).build()); + admin1.clusters().createCluster(cluster, ClusterData.builder().serviceUrl(pulsar1.getWebServiceAddress()).build()); admin1.tenants().createTenant(tenant, new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet(cluster))); admin1.namespaces().createNamespace(namespace, 16); - + String topicLookup = admin1.lookups().lookupTopic(topic); String bundleRange = admin1.lookups().getBundleRange(topic); - + String brokerServiceUrl = pulsar1.getBrokerServiceUrl(); - String brokerUrl = pulsar1.getSafeWebServiceAddress(); + String brokerId = pulsar1.getBrokerId(); log.debug("initial broker service url - {}", topicLookup); Random rand=new Random(); - + if (topicLookup.equals(brokerServiceUrl)) { int x = rand.nextInt(2); if (x == 0) { - brokerUrl = pulsar2.getSafeWebServiceAddress(); + brokerId = pulsar2.getBrokerId(); brokerServiceUrl = pulsar2.getBrokerServiceUrl(); } else { - brokerUrl = pulsar3.getSafeWebServiceAddress(); + brokerId = pulsar3.getBrokerId(); brokerServiceUrl = pulsar3.getBrokerServiceUrl(); } } - brokerUrl = brokerUrl.replaceFirst("http[s]?://", ""); - log.debug("destination broker service url - {}, broker url - {}", brokerServiceUrl, brokerUrl); - String leaderServiceUrl = admin1.brokers().getLeaderBroker().getServiceUrl(); - log.debug("leader serviceUrl - {}, broker1 service url - {}", leaderServiceUrl, pulsar1.getSafeWebServiceAddress()); - //Make a call to broker which is not a leader - if (!leaderServiceUrl.equals(pulsar1.getSafeWebServiceAddress())) { - admin1.namespaces().unloadNamespaceBundle(namespace, bundleRange, brokerUrl); + log.debug("destination broker service url - {}, broker url - {}", brokerServiceUrl, brokerId); + String leaderBrokerId = admin1.brokers().getLeaderBroker().getBrokerId(); + log.debug("leader lookup address - {}, broker1 lookup address - {}", leaderBrokerId, + pulsar1.getBrokerId()); + // Make a call to broker which is not a leader + if (!leaderBrokerId.equals(pulsar1.getBrokerId())) { + admin1.namespaces().unloadNamespaceBundle(namespace, bundleRange, brokerId); } else { - admin2.namespaces().unloadNamespaceBundle(namespace, bundleRange, brokerUrl); + admin2.namespaces().unloadNamespaceBundle(namespace, bundleRange, brokerId); } - + sleep(2000); String topicLookupAfterUnload = admin1.lookups().lookupTopic(topic); log.debug("final broker service url - {}", topicLookupAfterUnload); @@ -431,9 +440,9 @@ public void testLoadShedding() throws Exception { pulsar1.getConfiguration().setLoadBalancerEnabled(true); final LoadData loadData = (LoadData) getField(primaryLoadManagerSpy, "loadData"); final Map brokerDataMap = loadData.getBrokerData(); - final BrokerData brokerDataSpy1 = spy(brokerDataMap.get(primaryHost)); + final BrokerData brokerDataSpy1 = spy(brokerDataMap.get(primaryBrokerId)); when(brokerDataSpy1.getLocalData()).thenReturn(localBrokerData); - brokerDataMap.put(primaryHost, brokerDataSpy1); + brokerDataMap.put(primaryBrokerId, brokerDataSpy1); // Need to update all the bundle data for the shredder to see the spy. primaryLoadManagerSpy.handleDataNotification(new Notification(NotificationType.Created, LoadManager.LOADBALANCE_BROKERS_ROOT + "/broker:8080")); @@ -451,7 +460,7 @@ public void testLoadShedding() throws Exception { verify(namespacesSpy1, Mockito.times(1)) .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); assertEquals(bundleReference.get(), mockBundleName(2)); - assertEquals(selectedBrokerRef.get().get(), secondaryHost); + assertEquals(selectedBrokerRef.get().get(), secondaryBrokerId); primaryLoadManagerSpy.doLoadShedding(); // Now less expensive bundle will be unloaded (normally other bundle would move off and nothing would be @@ -459,13 +468,13 @@ public void testLoadShedding() throws Exception { verify(namespacesSpy1, Mockito.times(2)) .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); assertEquals(bundleReference.get(), mockBundleName(1)); - assertEquals(selectedBrokerRef.get().get(), secondaryHost); + assertEquals(selectedBrokerRef.get().get(), secondaryBrokerId); primaryLoadManagerSpy.doLoadShedding(); // Now both are in grace period: neither should be unloaded. verify(namespacesSpy1, Mockito.times(2)) .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); - assertEquals(selectedBrokerRef.get().get(), secondaryHost); + assertEquals(selectedBrokerRef.get().get(), secondaryBrokerId); // Test bundle transfer to same broker @@ -474,13 +483,11 @@ public void testLoadShedding() throws Exception { verify(namespacesSpy1, Mockito.times(3)) .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); - doReturn(Optional.of(primaryHost)).when(primaryLoadManagerSpy).selectBroker(any()); loadData.getRecentlyUnloadedBundles().clear(); primaryLoadManagerSpy.doLoadShedding(); // The bundle shouldn't be unloaded because the broker is the same. - verify(namespacesSpy1, Mockito.times(3)) + verify(namespacesSpy1, Mockito.times(4)) .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); - } // Test that ModularLoadManagerImpl will determine that writing local data to ZooKeeper is necessary if certain @@ -597,10 +604,12 @@ public void testNamespaceIsolationPoliciesForPrimaryAndSecondaryBrokers() throws final String tenant = "my-property"; final String cluster = "use"; final String namespace = "my-ns"; - final String broker1Address = pulsar1.getAdvertisedAddress() + "0"; - final String broker2Address = pulsar2.getAdvertisedAddress() + "1"; - final String sharedBroker = "broker3"; - admin1.clusters().createCluster(cluster, ClusterData.builder().serviceUrl("http://" + pulsar1.getAdvertisedAddress()).build()); + String broker1Host = pulsar1.getAdvertisedAddress() + "0"; + final String broker1Address = broker1Host + ":8080"; + String broker2Host = pulsar2.getAdvertisedAddress() + "1"; + final String broker2Address = broker2Host + ":8080"; + final String sharedBroker = "broker3:8080"; + admin1.clusters().createCluster(cluster, ClusterData.builder().serviceUrl(pulsar1.getWebServiceAddress()).build()); admin1.tenants().createTenant(tenant, new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet(cluster))); admin1.namespaces().createNamespace(tenant + "/" + cluster + "/" + namespace); @@ -608,8 +617,8 @@ public void testNamespaceIsolationPoliciesForPrimaryAndSecondaryBrokers() throws // set a new policy String newPolicyJsonTemplate = "{\"namespaces\":[\"%s/%s/%s.*\"],\"primary\":[\"%s\"]," + "\"secondary\":[\"%s\"],\"auto_failover_policy\":{\"policy_type\":\"min_available\",\"parameters\":{\"min_limit\":%s,\"usage_threshold\":80}}}"; - String newPolicyJson = String.format(newPolicyJsonTemplate, tenant, cluster, namespace, broker1Address, - broker2Address, 1); + String newPolicyJson = String.format(newPolicyJsonTemplate, tenant, cluster, namespace, broker1Host, + broker2Host, 1); String newPolicyName = "my-ns-isolation-policies"; ObjectMapper jsonMapper = ObjectMapperFactory.create(); NamespaceIsolationDataImpl nsPolicyData = jsonMapper.readValue(newPolicyJson.getBytes(), @@ -621,12 +630,12 @@ public void testNamespaceIsolationPoliciesForPrimaryAndSecondaryBrokers() throws ServiceUnitId serviceUnit = LoadBalancerTestingUtils.makeBundles(nsFactory, tenant, cluster, namespace, 1)[0]; BrokerTopicLoadingPredicate brokerTopicLoadingPredicate = new BrokerTopicLoadingPredicate() { @Override - public boolean isEnablePersistentTopics(String brokerUrl) { + public boolean isEnablePersistentTopics(String brokerId) { return true; } @Override - public boolean isEnableNonPersistentTopics(String brokerUrl) { + public boolean isEnableNonPersistentTopics(String brokerId) { return true; } }; @@ -658,8 +667,8 @@ public boolean isEnableNonPersistentTopics(String brokerUrl) { // (2) now we will have isolation policy : primary=broker1, secondary=broker2, minLimit=2 - newPolicyJson = String.format(newPolicyJsonTemplate, tenant, cluster, namespace, broker1Address, - broker2Address, 2); + newPolicyJson = String.format(newPolicyJsonTemplate, tenant, cluster, namespace, broker1Host, + broker2Host, 2); nsPolicyData = jsonMapper.readValue(newPolicyJson.getBytes(), NamespaceIsolationDataImpl.class); admin1.clusters().createNamespaceIsolationPolicy("use", newPolicyName, nsPolicyData); @@ -696,16 +705,18 @@ public void testLoadSheddingWithNamespaceIsolationPolicies() throws Exception { final String tenant = "my-tenant"; final String namespace = "my-tenant/use/my-ns"; final String bundle = "0x00000000_0xffffffff"; - final String brokerAddress = pulsar1.getAdvertisedAddress(); - final String broker1Address = pulsar1.getAdvertisedAddress() + 1; + final String brokerHost = pulsar1.getAdvertisedAddress(); + final String brokerAddress = brokerHost + ":8080"; + final String broker1Host = pulsar1.getAdvertisedAddress() + "1"; + final String broker1Address = broker1Host + ":8080"; - admin1.clusters().createCluster(cluster, ClusterData.builder().serviceUrl("http://" + pulsar1.getAdvertisedAddress()).build()); + admin1.clusters().createCluster(cluster, ClusterData.builder().serviceUrl(pulsar1.getWebServiceAddress()).build()); admin1.tenants().createTenant(tenant, new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet(cluster))); admin1.namespaces().createNamespace(namespace); @Cleanup - PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsar1.getSafeWebServiceAddress()).build(); + PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsar1.getWebServiceAddress()).build(); Producer producer = pulsarClient.newProducer().topic("persistent://" + namespace + "/my-topic1") .create(); ModularLoadManagerImpl loadManager = (ModularLoadManagerImpl) ((ModularLoadManagerWrapper) pulsar1 @@ -714,12 +725,13 @@ public void testLoadSheddingWithNamespaceIsolationPolicies() throws Exception { loadManager.updateAll(); // test1: no isolation policy - assertTrue(loadManager.shouldNamespacePoliciesUnload(namespace, bundle, primaryHost)); + assertTrue(loadManager.shouldNamespacePoliciesUnload(namespace, bundle, primaryBrokerId)); // test2: as isolation policy, there are not another broker to load the bundle. String newPolicyJsonTemplate = "{\"namespaces\":[\"%s.*\"],\"primary\":[\"%s\"]," + "\"secondary\":[\"%s\"],\"auto_failover_policy\":{\"policy_type\":\"min_available\",\"parameters\":{\"min_limit\":%s,\"usage_threshold\":80}}}"; - String newPolicyJson = String.format(newPolicyJsonTemplate, namespace, broker1Address,broker1Address, 1); + + String newPolicyJson = String.format(newPolicyJsonTemplate, namespace, broker1Host, broker1Host, 1); String newPolicyName = "my-ns-isolation-policies"; ObjectMapper jsonMapper = ObjectMapperFactory.create(); NamespaceIsolationDataImpl nsPolicyData = jsonMapper.readValue(newPolicyJson.getBytes(), @@ -728,11 +740,11 @@ public void testLoadSheddingWithNamespaceIsolationPolicies() throws Exception { assertFalse(loadManager.shouldNamespacePoliciesUnload(namespace, bundle, broker1Address)); // test3: as isolation policy, there are another can load the bundle. - String newPolicyJson1 = String.format(newPolicyJsonTemplate, namespace, brokerAddress,brokerAddress, 1); + String newPolicyJson1 = String.format(newPolicyJsonTemplate, namespace, brokerHost, brokerHost, 1); NamespaceIsolationDataImpl nsPolicyData1 = jsonMapper.readValue(newPolicyJson1.getBytes(), NamespaceIsolationDataImpl.class); admin1.clusters().updateNamespaceIsolationPolicy(cluster, newPolicyName, nsPolicyData1); - assertTrue(loadManager.shouldNamespacePoliciesUnload(namespace, bundle, primaryHost)); + assertTrue(loadManager.shouldNamespacePoliciesUnload(namespace, bundle, primaryBrokerId)); producer.close(); } @@ -749,7 +761,7 @@ public void testOwnBrokerZnodeByMultipleBroker() throws Exception { ServiceConfiguration config = new ServiceConfiguration(); config.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); config.setClusterName("use"); - config.setWebServicePort(Optional.of(0)); + config.setWebServicePort(Optional.of(PortManager.nextLockedFreePort())); config.setMetadataStoreUrl("zk:127.0.0.1:" + bkEnsemble.getZookeeperPort()); config.setBrokerShutdownTimeoutMs(0L); config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); @@ -757,10 +769,12 @@ public void testOwnBrokerZnodeByMultipleBroker() throws Exception { PulsarService pulsar = new PulsarService(config); // create znode using different zk-session final String brokerZnode = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + pulsar.getAdvertisedAddress() + ":" - + config.getWebServicePort(); - pulsar1.getLocalMetadataStore().put(brokerZnode, new byte[0], Optional.empty(), EnumSet.of(CreateOption.Ephemeral)).join(); + + config.getWebServicePort().get(); + pulsar1.getLocalMetadataStore() + .put(brokerZnode, new byte[0], Optional.empty(), EnumSet.of(CreateOption.Ephemeral)).join(); try { pulsar.start(); + fail("should have failed"); } catch (PulsarServerException e) { //Ok. } @@ -799,7 +813,7 @@ public void testBundleDataDefaultValue(boolean isV1) throws Exception { final String tenant = "my-tenant"; final String namespace = "my-ns"; NamespaceName ns = isV1 ? NamespaceName.get(tenant, cluster, namespace) : NamespaceName.get(tenant, namespace); - admin1.clusters().createCluster(cluster, ClusterData.builder().serviceUrl("http://" + pulsar1.getAdvertisedAddress()).build()); + admin1.clusters().createCluster(cluster, ClusterData.builder().serviceUrl(pulsar1.getWebServiceAddress()).build()); admin1.tenants().createTenant(tenant, new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet(cluster))); admin1.namespaces().createNamespace(ns.toString(), 16); @@ -848,7 +862,7 @@ public void testRemoveNonExistBundleData() final String topicName = tenant + "/" + namespace + "/" + "topic"; int bundleNumbers = 8; - admin1.clusters().createCluster(cluster, ClusterData.builder().serviceUrl("http://" + pulsar1.getAdvertisedAddress()).build()); + admin1.clusters().createCluster(cluster, ClusterData.builder().serviceUrl(pulsar1.getWebServiceAddress()).build()); admin1.tenants().createTenant(tenant, new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet(cluster))); admin1.namespaces().createNamespace(tenant + "/" + namespace, bundleNumbers); @@ -896,6 +910,9 @@ public void testRemoveNonExistBundleData() String topicToFindBundle = topicName + 0; NamespaceBundle bundleWillBeSplit = pulsar1.getNamespaceService().getBundle(TopicName.get(topicToFindBundle)); + final Optional leastLoaded = loadManagerWrapper.getLeastLoaded(bundleWillBeSplit); + assertFalse(leastLoaded.isEmpty()); + String bundleDataPath = BUNDLE_DATA_BASE_PATH + "/" + tenant + "/" + namespace; CompletableFuture> children = bundlesCache.getChildren(bundleDataPath); List bundles = children.join(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java index c22e49e5fea80..a0313ef743667 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java @@ -353,14 +353,14 @@ public void testUnloadNamespaceBundleWithStuckTopic() throws Exception { @Test public void testLoadReportDeserialize() throws Exception { - final String candidateBroker1 = "http://localhost:8000"; - final String candidateBroker2 = "http://localhost:3000"; - LoadReport lr = new LoadReport(null, null, candidateBroker1, null); - LocalBrokerData ld = new LocalBrokerData(null, null, candidateBroker2, null); - URI uri1 = new URI(candidateBroker1); - URI uri2 = new URI(candidateBroker2); - String path1 = String.format("%s/%s:%s", LoadManager.LOADBALANCE_BROKERS_ROOT, uri1.getHost(), uri1.getPort()); - String path2 = String.format("%s/%s:%s", LoadManager.LOADBALANCE_BROKERS_ROOT, uri2.getHost(), uri2.getPort()); + final String candidateBroker1 = "localhost:8000"; + String broker1Url = "pulsar://localhost:6650"; + final String candidateBroker2 = "localhost:3000"; + String broker2Url = "pulsar://localhost:6660"; + LoadReport lr = new LoadReport("http://" + candidateBroker1, null, broker1Url, null); + LocalBrokerData ld = new LocalBrokerData("http://" + candidateBroker2, null, broker2Url, null); + String path1 = String.format("%s/%s", LoadManager.LOADBALANCE_BROKERS_ROOT, candidateBroker1); + String path2 = String.format("%s/%s", LoadManager.LOADBALANCE_BROKERS_ROOT, candidateBroker2); pulsar.getLocalMetadataStore().put(path1, ObjectMapperFactory.getMapper().writer().writeValueAsBytes(lr), @@ -379,23 +379,23 @@ public void testLoadReportDeserialize() throws Exception { .getAndSet(new ModularLoadManagerWrapper(new ModularLoadManagerImpl())); oldLoadManager.stop(); LookupResult result2 = pulsar.getNamespaceService().createLookupResult(candidateBroker2, false, null).get(); - Assert.assertEquals(result1.getLookupData().getBrokerUrl(), candidateBroker1); - Assert.assertEquals(result2.getLookupData().getBrokerUrl(), candidateBroker2); + Assert.assertEquals(result1.getLookupData().getBrokerUrl(), broker1Url); + Assert.assertEquals(result2.getLookupData().getBrokerUrl(), broker2Url); System.out.println(result2); } @Test public void testCreateLookupResult() throws Exception { - final String candidateBroker = "pulsar://localhost:6650"; + final String candidateBroker = "localhost:8080"; + final String brokerUrl = "pulsar://localhost:6650"; final String listenerUrl = "pulsar://localhost:7000"; final String listenerUrlTls = "pulsar://localhost:8000"; final String listener = "listenerName"; Map advertisedListeners = new HashMap<>(); advertisedListeners.put(listener, AdvertisedListener.builder().brokerServiceUrl(new URI(listenerUrl)).brokerServiceUrlTls(new URI(listenerUrlTls)).build()); - LocalBrokerData ld = new LocalBrokerData(null, null, candidateBroker, null, advertisedListeners); - URI uri = new URI(candidateBroker); - String path = String.format("%s/%s:%s", LoadManager.LOADBALANCE_BROKERS_ROOT, uri.getHost(), uri.getPort()); + LocalBrokerData ld = new LocalBrokerData("http://" + candidateBroker, null, brokerUrl, null, advertisedListeners); + String path = String.format("%s/%s", LoadManager.LOADBALANCE_BROKERS_ROOT, candidateBroker); pulsar.getLocalMetadataStore().put(path, ObjectMapperFactory.getMapper().writer().writeValueAsBytes(ld), @@ -405,7 +405,7 @@ public void testCreateLookupResult() throws Exception { LookupResult noListener = pulsar.getNamespaceService().createLookupResult(candidateBroker, false, null).get(); LookupResult withListener = pulsar.getNamespaceService().createLookupResult(candidateBroker, false, listener).get(); - Assert.assertEquals(noListener.getLookupData().getBrokerUrl(), candidateBroker); + Assert.assertEquals(noListener.getLookupData().getBrokerUrl(), brokerUrl); Assert.assertEquals(withListener.getLookupData().getBrokerUrl(), listenerUrl); Assert.assertEquals(withListener.getLookupData().getBrokerUrlTls(), listenerUrlTls); System.out.println(withListener); @@ -687,7 +687,7 @@ public void testSplitBundleWithHighestThroughput() throws Exception { @Test public void testHeartbeatNamespaceMatch() throws Exception { - NamespaceName namespaceName = NamespaceService.getHeartbeatNamespace(pulsar.getLookupServiceAddress(), conf); + NamespaceName namespaceName = NamespaceService.getHeartbeatNamespace(pulsar.getBrokerId(), conf); NamespaceBundle namespaceBundle = pulsar.getNamespaceService().getNamespaceBundleFactory().getFullBundle(namespaceName); assertTrue(NamespaceService.isSystemServiceNamespace( NamespaceBundle.getBundleNamespace(namespaceBundle.toString()))); @@ -707,7 +707,7 @@ public void testModularLoadManagerRemoveInactiveBundleFromLoadData() throws Exce Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager"); loadManagerField.setAccessible(true); doReturn(true).when(loadManager).isCentralized(); - SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getSafeWebServiceAddress(), null); + SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getBrokerId(), null); Optional res = Optional.of(resourceUnit); doReturn(res).when(loadManager).getLeastLoaded(any(ServiceUnitId.class)); loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager)); @@ -860,10 +860,7 @@ private void waitResourceDataUpdateToZK(LoadManager loadManager) throws Exceptio public CompletableFuture registryBrokerDataChangeNotice() { CompletableFuture completableFuture = new CompletableFuture<>(); - String lookupServiceAddress = pulsar.getAdvertisedAddress() + ":" - + (conf.getWebServicePort().isPresent() ? conf.getWebServicePort().get() - : conf.getWebServicePortTls().get()); - String brokerDataPath = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + lookupServiceAddress; + String brokerDataPath = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + pulsar.getBrokerId(); pulsar.getLocalMetadataStore().registerListener(notice -> { if (brokerDataPath.equals(notice.getPath())){ if (!completableFuture.isDone()) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AdvertisedAddressTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AdvertisedAddressTest.java index 554c663850fd9..19e40ebf9960f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AdvertisedAddressTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AdvertisedAddressTest.java @@ -75,7 +75,7 @@ public void testAdvertisedAddress() throws Exception { Assert.assertEquals( pulsar.getAdvertisedAddress(), advertisedAddress ); Assert.assertEquals( pulsar.getBrokerServiceUrl(), String.format("pulsar://%s:%d", advertisedAddress, pulsar.getBrokerListenPort().get()) ); Assert.assertEquals( pulsar.getSafeWebServiceAddress(), String.format("http://%s:%d", advertisedAddress, pulsar.getListenPortHTTP().get()) ); - String brokerZkPath = String.format("/loadbalance/brokers/%s:%d", pulsar.getAdvertisedAddress(), pulsar.getListenPortHTTP().get()); + String brokerZkPath = String.format("/loadbalance/brokers/%s", pulsar.getBrokerId()); String bkBrokerData = new String(bkEnsemble.getZkClient().getData(brokerZkPath, false, new Stat()), StandardCharsets.UTF_8); JsonObject jsonBkBrokerData = new Gson().fromJson(bkBrokerData, JsonObject.class); Assert.assertEquals( jsonBkBrokerData.get("pulsarServiceUrl").getAsString(), pulsar.getBrokerServiceUrl() ); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java index b2fbe824b3305..8e902d5d1e700 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java @@ -18,8 +18,17 @@ */ package org.apache.pulsar.broker.service; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; +import com.carrotsearch.hppc.ObjectSet; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -28,10 +37,14 @@ import lombok.Cleanup; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.ConsumerBuilder; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageIdAdv; @@ -39,8 +52,10 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.BatchMessageIdImpl; import org.apache.pulsar.client.impl.ConsumerImpl; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.collections.BitSetRecyclable; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.BeforeClass; @@ -401,4 +416,256 @@ public void testMixIndexAndNonIndexUnAckMessageCount() throws Exception { assertEquals(admin.topics().getStats(topicName).getSubscriptions() .get("sub").getUnackedMessages(), 0); } + + @Test + public void testUnAckMessagesWhenConcurrentDeliveryAndAck() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://prop/ns-abc/tp"); + final String subName = "s1"; + final int receiverQueueSize = 500; + admin.topics().createNonPartitionedTopic(topicName); + admin.topics().createSubscription(topicName, subName, MessageId.earliest); + ConsumerBuilder consumerBuilder = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .receiverQueueSize(receiverQueueSize) + .subscriptionName(subName) + .enableBatchIndexAcknowledgment(true) + .subscriptionType(SubscriptionType.Shared) + .isAckReceiptEnabled(true); + + // Send 100 messages. + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .enableBatching(true) + .batchingMaxPublishDelay(1, TimeUnit.HOURS) + .create(); + CompletableFuture lastSent = null; + for (int i = 0; i < 100; i++) { + lastSent = producer.sendAsync(i + ""); + } + producer.flush(); + lastSent.join(); + + // When consumer1 is closed, may some messages are in the client memory(it they are being acked now). + Consumer consumer1 = consumerBuilder.consumerName("c1").subscribe(); + Message[] messagesInClientMemory = new Message[2]; + for (int i = 0; i < 2; i++) { + Message msg = consumer1.receive(2, TimeUnit.SECONDS); + assertNotNull(msg); + messagesInClientMemory[i] = msg; + } + ConsumerImpl consumer2 = (ConsumerImpl) consumerBuilder.consumerName("c2").subscribe(); + Awaitility.await().until(() -> consumer2.isConnected()); + + // The consumer2 will receive messages after consumer1 closed. + // Insert a delay mechanism to make the flow like below: + // 1. Close consumer1, then the 100 messages will be redelivered. + // 2. Read redeliver messages. No messages were acked at this time. + // 3. The in-flight ack of two messages is finished. + // 4. Send the messages to consumer2, consumer2 will get all the 100 messages. + CompletableFuture receiveMessageSignal2 = new CompletableFuture<>(); + org.apache.pulsar.broker.service.Consumer serviceConsumer2 = + makeConsumerReceiveMessagesDelay(topicName, subName, "c2", receiveMessageSignal2); + // step 1: close consumer. + consumer1.close(); + // step 2: wait for read messages from replay queue. + Thread.sleep(2 * 1000); + // step 3: wait for the in-flight ack. + BitSetRecyclable bitSetRecyclable = createBitSetRecyclable(100); + long ledgerId = 0, entryId = 0; + for (Message message : messagesInClientMemory) { + BatchMessageIdImpl msgId = (BatchMessageIdImpl) message.getMessageId(); + bitSetRecyclable.clear(msgId.getBatchIndex()); + ledgerId = msgId.getLedgerId(); + entryId = msgId.getEntryId(); + } + getCursor(topicName, subName).delete(PositionImpl.get(ledgerId, entryId, bitSetRecyclable.toLongArray())); + // step 4: send messages to consumer2. + receiveMessageSignal2.complete(null); + // Verify: Consumer2 will get all the 100 messages, and "unAckMessages" is 100. + List messages2 = new ArrayList<>(); + while (true) { + Message msg = consumer2.receive(2, TimeUnit.SECONDS); + if (msg == null) { + break; + } + messages2.add(msg); + } + assertEquals(messages2.size(), 100); + assertEquals(serviceConsumer2.getUnackedMessages(), 100); + // After the messages were pop out, the permits in the client memory went to 100. + Awaitility.await().untilAsserted(() -> { + assertEquals(serviceConsumer2.getAvailablePermits() + consumer2.getAvailablePermits(), + receiverQueueSize); + }); + + // cleanup. + producer.close(); + consumer2.close(); + admin.topics().delete(topicName, false); + } + + private BitSetRecyclable createBitSetRecyclable(int batchSize) { + BitSetRecyclable bitSetRecyclable = new BitSetRecyclable(batchSize); + for (int i = 0; i < batchSize; i++) { + bitSetRecyclable.set(i); + } + return bitSetRecyclable; + } + + private ManagedCursorImpl getCursor(String topic, String sub) { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topic, false).join().get(); + PersistentDispatcherMultipleConsumers dispatcher = + (PersistentDispatcherMultipleConsumers) persistentTopic.getSubscription(sub).getDispatcher(); + return (ManagedCursorImpl) dispatcher.getCursor(); + } + + /*** + * After {@param signal} complete, the consumer({@param consumerName}) start to receive messages. + */ + private org.apache.pulsar.broker.service.Consumer makeConsumerReceiveMessagesDelay(String topic, String sub, + String consumerName, + CompletableFuture signal) throws Exception { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topic, false).join().get(); + PersistentDispatcherMultipleConsumers dispatcher = + (PersistentDispatcherMultipleConsumers) persistentTopic.getSubscription(sub).getDispatcher(); + org.apache.pulsar.broker.service.Consumer serviceConsumer = null; + for (org.apache.pulsar.broker.service.Consumer c : dispatcher.getConsumers()){ + if (c.consumerName().equals(consumerName)) { + serviceConsumer = c; + break; + } + } + final org.apache.pulsar.broker.service.Consumer originalConsumer = serviceConsumer; + + // Insert a delay signal. + org.apache.pulsar.broker.service.Consumer spyServiceConsumer = spy(originalConsumer); + doAnswer(invocation -> { + List entries = (List) invocation.getArguments()[0]; + EntryBatchSizes batchSizes = (EntryBatchSizes) invocation.getArguments()[1]; + EntryBatchIndexesAcks batchIndexesAcks = (EntryBatchIndexesAcks) invocation.getArguments()[2]; + int totalMessages = (int) invocation.getArguments()[3]; + long totalBytes = (long) invocation.getArguments()[4]; + long totalChunkedMessages = (long) invocation.getArguments()[5]; + RedeliveryTracker redeliveryTracker = (RedeliveryTracker) invocation.getArguments()[6]; + return signal.thenApply(__ -> originalConsumer.sendMessages(entries, batchSizes, batchIndexesAcks, totalMessages, totalBytes, + totalChunkedMessages, redeliveryTracker)).join(); + }).when(spyServiceConsumer) + .sendMessages(anyList(), any(), any(), anyInt(), anyLong(), anyLong(), any()); + doAnswer(invocation -> { + List entries = (List) invocation.getArguments()[0]; + EntryBatchSizes batchSizes = (EntryBatchSizes) invocation.getArguments()[1]; + EntryBatchIndexesAcks batchIndexesAcks = (EntryBatchIndexesAcks) invocation.getArguments()[2]; + int totalMessages = (int) invocation.getArguments()[3]; + long totalBytes = (long) invocation.getArguments()[4]; + long totalChunkedMessages = (long) invocation.getArguments()[5]; + RedeliveryTracker redeliveryTracker = (RedeliveryTracker) invocation.getArguments()[6]; + long epoch = (long) invocation.getArguments()[7]; + return signal.thenApply(__ -> originalConsumer.sendMessages(entries, batchSizes, batchIndexesAcks, totalMessages, totalBytes, + totalChunkedMessages, redeliveryTracker, epoch)).join(); + }).when(spyServiceConsumer) + .sendMessages(anyList(), any(), any(), anyInt(), anyLong(), anyLong(), any(), anyLong()); + + // Replace the consumer. + Field fConsumerList = AbstractDispatcherMultipleConsumers.class.getDeclaredField("consumerList"); + Field fConsumerSet = AbstractDispatcherMultipleConsumers.class.getDeclaredField("consumerSet"); + fConsumerList.setAccessible(true); + fConsumerSet.setAccessible(true); + List consumerList = + (List) fConsumerList.get(dispatcher); + ObjectSet consumerSet = + (ObjectSet) fConsumerSet.get(dispatcher); + + consumerList.remove(originalConsumer); + consumerSet.removeAll(originalConsumer); + consumerList.add(spyServiceConsumer); + consumerSet.add(spyServiceConsumer); + return originalConsumer; + } + + /*** + * 1. Send a batch message contains 100 single messages. + * 2. Ack 2 messages. + * 3. Redeliver the batch message and ack them. + * 4. Verify: the permits is correct. + */ + @Test + public void testPermitsIfHalfAckBatchMessage() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://prop/ns-abc/tp"); + final String subName = "s1"; + final int receiverQueueSize = 1000; + final int ackedMessagesCountInTheFistStep = 2; + admin.topics().createNonPartitionedTopic(topicName); + admin.topics(). createSubscription(topicName, subName, MessageId.earliest); + ConsumerBuilder consumerBuilder = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .receiverQueueSize(receiverQueueSize) + .subscriptionName(subName) + .enableBatchIndexAcknowledgment(true) + .subscriptionType(SubscriptionType.Shared) + .isAckReceiptEnabled(true); + + // Send 100 messages. + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .enableBatching(true) + .batchingMaxPublishDelay(1, TimeUnit.HOURS) + .create(); + CompletableFuture lastSent = null; + for (int i = 1; i <= 100; i++) { + lastSent = producer. sendAsync(i + ""); + } + producer.flush(); + lastSent.join(); + + // Ack 2 messages, and trigger a redelivery. + Consumer consumer1 = consumerBuilder.subscribe(); + for (int i = 0; i < ackedMessagesCountInTheFistStep; i++) { + Message msg = consumer1. receive(2, TimeUnit.SECONDS); + assertNotNull(msg); + consumer1.acknowledge(msg); + } + consumer1.close(); + + // Receive the left 98 messages, and ack them. + // Verify the permits is correct. + ConsumerImpl consumer2 = (ConsumerImpl) consumerBuilder.subscribe(); + Awaitility.await().until(() -> consumer2.isConnected()); + List messages = new ArrayList<>(); + int nextMessageValue = ackedMessagesCountInTheFistStep + 1; + while (true) { + Message msg = consumer2.receive(2, TimeUnit.SECONDS); + if (msg == null) { + break; + } + assertEquals(msg.getValue(), nextMessageValue + ""); + messages.add(msg.getMessageId()); + nextMessageValue++; + } + assertEquals(messages.size(), 98); + consumer2.acknowledge(messages); + + org.apache.pulsar.broker.service.Consumer serviceConsumer2 = + getTheUniqueServiceConsumer(topicName, subName); + Awaitility.await().untilAsserted(() -> { + // After the messages were pop out, the permits in the client memory went to 98. + int permitsInClientMemory = consumer2.getAvailablePermits(); + int permitsInBroker = serviceConsumer2.getAvailablePermits(); + assertEquals(permitsInClientMemory + permitsInBroker, receiverQueueSize); + }); + + // cleanup. + producer.close(); + consumer2.close(); + admin.topics().delete(topicName, false); + } + + private org.apache.pulsar.broker.service.Consumer getTheUniqueServiceConsumer(String topic, String sub) { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService(). getTopic(topic, false).join().get(); + PersistentDispatcherMultipleConsumers dispatcher = + (PersistentDispatcherMultipleConsumers) persistentTopic.getSubscription(sub).getDispatcher(); + return dispatcher.getConsumers().iterator().next(); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java index f456a133d99b1..b6a73274f440b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java @@ -1534,9 +1534,9 @@ public void testIsSystemTopic() { assertTrue(brokerService.isSystemTopic(TRANSACTION_COORDINATOR_ASSIGN)); assertTrue(brokerService.isSystemTopic(TRANSACTION_COORDINATOR_LOG)); NamespaceName heartbeatNamespaceV1 = NamespaceService - .getHeartbeatNamespace(pulsar.getLookupServiceAddress(), pulsar.getConfig()); + .getHeartbeatNamespace(pulsar.getBrokerId(), pulsar.getConfig()); NamespaceName heartbeatNamespaceV2 = NamespaceService - .getHeartbeatNamespaceV2(pulsar.getLookupServiceAddress(), pulsar.getConfig()); + .getHeartbeatNamespaceV2(pulsar.getBrokerId(), pulsar.getConfig()); assertTrue(brokerService.isSystemTopic("persistent://" + heartbeatNamespaceV1.toString() + "/healthcheck")); assertTrue(brokerService.isSystemTopic(heartbeatNamespaceV2.toString() + "/healthcheck")); } @@ -1724,13 +1724,4 @@ public void testUnsubscribeNonDurableSub() throws Exception { fail("Unsubscribe failed"); } } - - @Test - public void testGetLookupServiceAddress() throws Exception { - cleanup(); - setup(); - conf.setWebServicePortTls(Optional.of(8081)); - assertEquals(pulsar.getLookupServiceAddress(), "localhost:8081"); - resetState(); - } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelectorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelectorTest.java index dbca31416bb9d..48311c57338b5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelectorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelectorTest.java @@ -21,18 +21,18 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; - -import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerAssignException; -import org.apache.pulsar.client.api.Range; -import org.testng.Assert; -import org.testng.annotations.Test; - import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerAssignException; +import org.apache.pulsar.client.api.Range; +import org.testng.Assert; +import org.testng.annotations.Test; @Test(groups = "broker") public class ConsistentHashingStickyKeyConsumerSelectorTest { @@ -154,17 +154,17 @@ public void testGetConsumerKeyHashRanges() throws BrokerServiceException.Consume } Map> expectedResult = new HashMap<>(); expectedResult.put(consumers.get(0), Arrays.asList( - Range.of(0, 330121749), - Range.of(330121750, 618146114), - Range.of(1797637922, 1976098885))); + Range.of(119056335, 242013991), + Range.of(722195657, 1656011842), + Range.of(1707482098, 1914695766))); expectedResult.put(consumers.get(1), Arrays.asList( - Range.of(938427576, 1094135919), - Range.of(1138613629, 1342907082), - Range.of(1342907083, 1797637921))); + Range.of(0, 90164503), + Range.of(90164504, 119056334), + Range.of(382436668, 722195656))); expectedResult.put(consumers.get(2), Arrays.asList( - Range.of(618146115, 772640562), - Range.of(772640563, 938427575), - Range.of(1094135920, 1138613628))); + Range.of(242013992, 242377547), + Range.of(242377548, 382436667), + Range.of(1656011843, 1707482097))); for (Map.Entry> entry : selector.getConsumerKeyHashRanges().entrySet()) { System.out.println(entry.getValue()); Assert.assertEquals(entry.getValue(), expectedResult.get(entry.getKey())); @@ -172,4 +172,48 @@ public void testGetConsumerKeyHashRanges() throws BrokerServiceException.Consume } Assert.assertEquals(expectedResult.size(), 0); } + + // reproduces https://github.com/apache/pulsar/issues/22050 + @Test + public void shouldNotCollideWithConsumerNameEndsWithNumber() { + ConsistentHashingStickyKeyConsumerSelector selector = new ConsistentHashingStickyKeyConsumerSelector(12); + List consumerName = Arrays.asList("consumer1", "consumer11"); + List consumers = new ArrayList<>(); + for (String s : consumerName) { + Consumer consumer = mock(Consumer.class); + when(consumer.consumerName()).thenReturn(s); + selector.addConsumer(consumer); + consumers.add(consumer); + } + Map rangeToConsumer = new HashMap<>(); + for (Map.Entry> entry : selector.getConsumerKeyHashRanges().entrySet()) { + for (Range range : entry.getValue()) { + Consumer previous = rangeToConsumer.put(range, entry.getKey()); + if (previous != null) { + Assert.fail("Ranges are colliding between " + previous.consumerName() + " and " + entry.getKey() + .consumerName()); + } + } + } + } + + @Test + public void shouldRemoveConsumersFromConsumerKeyHashRanges() { + ConsistentHashingStickyKeyConsumerSelector selector = new ConsistentHashingStickyKeyConsumerSelector(12); + List consumers = IntStream.range(1, 100).mapToObj(i -> "consumer" + i) + .map(consumerName -> { + Consumer consumer = mock(Consumer.class); + when(consumer.consumerName()).thenReturn(consumerName); + return consumer; + }).collect(Collectors.toList()); + + // when consumers are added + consumers.forEach(selector::addConsumer); + // then each consumer should have a range + Assert.assertEquals(selector.getConsumerKeyHashRanges().size(), consumers.size()); + // when consumers are removed + consumers.forEach(selector::removeConsumer); + // then there should be no mapping remaining + Assert.assertEquals(selector.getConsumerKeyHashRanges().size(), 0); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/InactiveTopicDeleteTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/InactiveTopicDeleteTest.java index 84c4670f2bb3b..1549ba8d81c09 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/InactiveTopicDeleteTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/InactiveTopicDeleteTest.java @@ -599,11 +599,11 @@ public void testHealthTopicInactiveNotClean() throws Exception { super.baseSetup(); // init topic NamespaceName heartbeatNamespaceV1 = NamespaceService - .getHeartbeatNamespace(pulsar.getLookupServiceAddress(), pulsar.getConfig()); + .getHeartbeatNamespace(pulsar.getBrokerId(), pulsar.getConfig()); final String healthCheckTopicV1 = "persistent://" + heartbeatNamespaceV1 + "/healthcheck"; NamespaceName heartbeatNamespaceV2 = NamespaceService - .getHeartbeatNamespaceV2(pulsar.getLookupServiceAddress(), pulsar.getConfig()); + .getHeartbeatNamespaceV2(pulsar.getBrokerId(), pulsar.getConfig()); final String healthCheckTopicV2 = "persistent://" + heartbeatNamespaceV2 + "/healthcheck"; admin.brokers().healthcheck(TopicVersion.V1); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java index 8269f40e60845..1accd04f4918c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java @@ -21,15 +21,21 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import java.lang.reflect.Field; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.service.persistent.PersistentReplicator; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.ProducerImpl; import org.apache.pulsar.common.policies.data.TopicStats; import org.awaitility.Awaitility; +import org.mockito.Mockito; +import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -49,6 +55,29 @@ public void cleanup() throws Exception { super.cleanup(); } + private void waitReplicatorStarted(String topicName) { + Awaitility.await().untilAsserted(() -> { + Optional topicOptional2 = pulsar2.getBrokerService().getTopic(topicName, false).get(); + assertTrue(topicOptional2.isPresent()); + PersistentTopic persistentTopic2 = (PersistentTopic) topicOptional2.get(); + assertFalse(persistentTopic2.getProducers().isEmpty()); + }); + } + + /** + * Override "AbstractReplicator.producer" by {@param producer} and return the original value. + */ + private ProducerImpl overrideProducerForReplicator(AbstractReplicator replicator, ProducerImpl newProducer) + throws Exception { + Field producerField = AbstractReplicator.class.getDeclaredField("producer"); + producerField.setAccessible(true); + ProducerImpl originalValue = (ProducerImpl) producerField.get(replicator); + synchronized (replicator) { + producerField.set(replicator, newProducer); + } + return originalValue; + } + @Test public void testReplicatorProducerStatInTopic() throws Exception { final String topicName = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp_"); @@ -79,18 +108,13 @@ public void testReplicatorProducerStatInTopic() throws Exception { public void testCreateRemoteConsumerFirst() throws Exception { final String topicName = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp_"); Producer producer1 = client1.newProducer(Schema.STRING).topic(topicName).create(); - // Wait for replicator started. - Awaitility.await().untilAsserted(() -> { - Optional topicOptional2 = pulsar2.getBrokerService().getTopic(topicName, false).get(); - assertTrue(topicOptional2.isPresent()); - PersistentTopic persistentTopic2 = (PersistentTopic) topicOptional2.get(); - assertFalse(persistentTopic2.getProducers().isEmpty()); - }); + // The topic in cluster2 has a replicator created producer(schema Auto_Produce), but does not have any schema。 // Verify: the consumer of this cluster2 can create successfully. Consumer consumer2 = client2.newConsumer(Schema.STRING).topic(topicName).subscriptionName("s1") .subscribe();; - + // Wait for replicator started. + waitReplicatorStarted(topicName); // cleanup. producer1.close(); consumer2.close(); @@ -99,4 +123,34 @@ public void testCreateRemoteConsumerFirst() throws Exception { admin2.topics().delete(topicName); }); } + + @Test + public void testTopicCloseWhenInternalProducerCloseErrorOnce() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp_"); + admin1.topics().createNonPartitionedTopic(topicName); + // Wait for replicator started. + waitReplicatorStarted(topicName); + PersistentTopic persistentTopic = + (PersistentTopic) pulsar1.getBrokerService().getTopic(topicName, false).join().get(); + PersistentReplicator replicator = + (PersistentReplicator) persistentTopic.getReplicators().values().iterator().next(); + // Mock an error when calling "replicator.disconnect()" + ProducerImpl mockProducer = Mockito.mock(ProducerImpl.class); + Mockito.when(mockProducer.closeAsync()).thenReturn(CompletableFuture.failedFuture(new Exception("mocked ex"))); + ProducerImpl originalProducer = overrideProducerForReplicator(replicator, mockProducer); + // Verify: since the "replicator.producer.closeAsync()" will retry after it failed, the topic unload should be + // successful. + admin1.topics().unload(topicName); + // Verify: After "replicator.producer.closeAsync()" retry again, the "replicator.producer" will be closed + // successful. + overrideProducerForReplicator(replicator, originalProducer); + Awaitility.await().untilAsserted(() -> { + Assert.assertFalse(replicator.isConnected()); + }); + // cleanup. + cleanupTopics(() -> { + admin1.topics().delete(topicName); + admin2.topics().delete(topicName); + }); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java index f0e2e6eafcdfb..ace552a55a72a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java @@ -81,8 +81,11 @@ public class PersistentMessageFinderTest extends MockedBookKeeperTestCase { public static byte[] createMessageWrittenToLedger(String msg) { + return createMessageWrittenToLedger(msg, System.currentTimeMillis()); + } + public static byte[] createMessageWrittenToLedger(String msg, long messageTimestamp) { MessageMetadata messageMetadata = new MessageMetadata() - .setPublishTime(System.currentTimeMillis()) + .setPublishTime(messageTimestamp) .setProducerName("createMessageWrittenToLedger") .setSequenceId(1); ByteBuf data = UnpooledByteBufAllocator.DEFAULT.heapBuffer().writeBytes(msg.getBytes()); @@ -437,6 +440,29 @@ void testMessageExpiryWithTimestampNonRecoverableException() throws Exception { } + @Test + public void testIncorrectClientClock() throws Exception { + final String ledgerAndCursorName = "testIncorrectClientClock"; + int maxTTLSeconds = 1; + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setMaxEntriesPerLedger(1); + ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open(ledgerAndCursorName, config); + ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor(ledgerAndCursorName); + // set client clock to 10 days later + long incorrectPublishTimestamp = System.currentTimeMillis() + TimeUnit.DAYS.toMillis(10); + for (int i = 0; i < 10; i++) { + ledger.addEntry(createMessageWrittenToLedger("msg" + i, incorrectPublishTimestamp)); + } + assertEquals(ledger.getLedgersInfoAsList().size(), 10); + PersistentTopic mock = mock(PersistentTopic.class); + when(mock.getName()).thenReturn("topicname"); + when(mock.getLastPosition()).thenReturn(PositionImpl.EARLIEST); + PersistentMessageExpiryMonitor monitor = new PersistentMessageExpiryMonitor(mock, c1.getName(), c1, null); + Thread.sleep(TimeUnit.SECONDS.toMillis(maxTTLSeconds)); + monitor.expireMessages(maxTTLSeconds); + assertEquals(c1.getNumberOfEntriesInBacklog(true), 0); + } + @Test void testMessageExpiryWithPosition() throws Exception { final String ledgerAndCursorName = "testPersistentMessageExpiryWithPositionNonRecoverableLedgers"; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorSubscriptionTest.java index fe519827be74a..4cc3a9ada7d04 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorSubscriptionTest.java @@ -52,6 +52,7 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.common.policies.data.PartitionedTopicStats; import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; @@ -868,6 +869,7 @@ public void testReplicatedSubscriptionWithCompaction() throws Exception { .topic(topicName) .subscriptionName("sub2") .subscriptionType(SubscriptionType.Exclusive) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) .readCompacted(true) .subscribe(); List result = new ArrayList<>(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java index 7e5cecd5796c2..88a668e8745d5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java @@ -83,6 +83,7 @@ import org.apache.pulsar.client.api.RawMessage; import org.apache.pulsar.client.api.RawReader; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.TypedMessageBuilder; import org.apache.pulsar.client.api.schema.GenericRecord; @@ -238,16 +239,14 @@ public void activeBrokerParse() throws Exception { pulsar1.getConfiguration().setAuthorizationEnabled(true); //init clusterData - String cluster2ServiceUrls = String.format("%s,localhost:1234,localhost:5678,localhost:5677,localhost:5676", - pulsar2.getWebServiceAddress()); - ClusterData cluster2Data = ClusterData.builder().serviceUrl(cluster2ServiceUrls).build(); + ClusterData cluster2Data = ClusterData.builder().serviceUrl(pulsar2.getWebServiceAddress()).build(); String cluster2 = "activeCLuster2"; admin2.clusters().createCluster(cluster2, cluster2Data); Awaitility.await().until(() -> admin2.clusters().getCluster(cluster2) != null); List list = admin1.brokers().getActiveBrokers(cluster2); - assertEquals(list.get(0), urlTls2.toString().replace("https://", "")); + assertEquals(list.get(0), pulsar2.getBrokerId()); //restore configuration pulsar1.getConfiguration().setAuthorizationEnabled(false); } @@ -1805,4 +1804,74 @@ public void testReplicatorProducerNotExceed() throws Exception { Assert.assertThrows(PulsarClientException.ProducerBusyException.class, () -> new MessageProducer(url2, dest2)); } + + @Test + public void testReplicatorWithTTL() throws Exception { + log.info("--- Starting ReplicatorTest::testReplicatorWithTTL ---"); + + final String cluster1 = pulsar1.getConfig().getClusterName(); + final String cluster2 = pulsar2.getConfig().getClusterName(); + final String namespace = BrokerTestUtil.newUniqueName("pulsar/ns"); + final TopicName topic = TopicName + .get(BrokerTestUtil.newUniqueName("persistent://" + namespace + "/testReplicatorWithTTL")); + admin1.namespaces().createNamespace(namespace, Sets.newHashSet(cluster1, cluster2)); + admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet(cluster1, cluster2)); + admin1.topics().createNonPartitionedTopic(topic.toString()); + admin1.topicPolicies().setMessageTTL(topic.toString(), 1); + + @Cleanup + PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS) + .build(); + + @Cleanup + Producer persistentProducer1 = client1.newProducer().topic(topic.toString()).create(); + persistentProducer1.send("V1".getBytes()); + + waitReplicateFinish(topic, admin1); + + PersistentTopic persistentTopic = + (PersistentTopic) pulsar1.getBrokerService().getTopicReference(topic.toString()).get(); + persistentTopic.getReplicators().forEach((cluster, replicator) -> { + PersistentReplicator persistentReplicator = (PersistentReplicator) replicator; + // Pause replicator + persistentReplicator.disconnect(); + }); + + persistentProducer1.send("V2".getBytes()); + persistentProducer1.send("V3".getBytes()); + + Thread.sleep(1000); + + admin1.topics().expireMessagesForAllSubscriptions(topic.toString(), 1); + + persistentTopic.getReplicators().forEach((cluster, replicator) -> { + PersistentReplicator persistentReplicator = (PersistentReplicator) replicator; + persistentReplicator.startProducer(); + }); + + waitReplicateFinish(topic, admin1); + + persistentProducer1.send("V4".getBytes()); + + waitReplicateFinish(topic, admin1); + + @Cleanup + PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()).statsInterval(0, TimeUnit.SECONDS) + .build(); + + @Cleanup + Consumer consumer = client2.newConsumer().topic(topic.toString()) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscriptionName("sub").subscribe(); + + List result = new ArrayList<>(); + while (true) { + Message receive = consumer.receive(2, TimeUnit.SECONDS); + if (receive == null) { + break; + } + result.add(new String(receive.getValue())); + } + + assertEquals(result, Lists.newArrayList("V1", "V2", "V3", "V4")); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index 919e82afa6415..1b9289042745c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -33,6 +33,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -43,6 +44,7 @@ import org.apache.pulsar.broker.service.BrokerServiceException.TopicPoliciesCacheNotInitException; import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.client.impl.BackoffBuilder; import org.apache.pulsar.common.events.PulsarEvent; @@ -67,6 +69,8 @@ public class SystemTopicBasedTopicPoliciesServiceTest extends MockedPulsarServic private static final String NAMESPACE2 = "system-topic/namespace-2"; private static final String NAMESPACE3 = "system-topic/namespace-3"; + private static final String NAMESPACE4 = "system-topic/namespace-4"; + private static final TopicName TOPIC1 = TopicName.get("persistent", NamespaceName.get(NAMESPACE1), "topic-1"); private static final TopicName TOPIC2 = TopicName.get("persistent", NamespaceName.get(NAMESPACE1), "topic-2"); private static final TopicName TOPIC3 = TopicName.get("persistent", NamespaceName.get(NAMESPACE2), "topic-1"); @@ -430,4 +434,35 @@ public void testGetTopicPoliciesWithCleanCache() throws Exception { result.join(); } + + @Test + public void testWriterCache() throws Exception { + admin.namespaces().createNamespace(NAMESPACE4); + for (int i = 1; i <= 5; i ++) { + final String topicName = "persistent://" + NAMESPACE4 + "/testWriterCache" + i; + admin.topics().createNonPartitionedTopic(topicName); + pulsarClient.newProducer(Schema.STRING).topic(topicName).create().close(); + } + @Cleanup("shutdown") + ExecutorService executorService = Executors.newFixedThreadPool(5); + for (int i = 1; i <= 5; i ++) { + int finalI = i; + executorService.execute(() -> { + final String topicName = "persistent://" + NAMESPACE4 + "/testWriterCache" + finalI; + try { + admin.topicPolicies().setMaxConsumers(topicName, 2); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + SystemTopicBasedTopicPoliciesService service = (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + Assert.assertNotNull(service.getWriterCaches().synchronous().get(NamespaceName.get(NAMESPACE4))); + for (int i = 1; i <= 5; i ++) { + final String topicName = "persistent://" + NAMESPACE4 + "/testWriterCache" + i; + admin.topics().delete(topicName); + } + admin.namespaces().deleteNamespace(NAMESPACE4); + Assert.assertNull(service.getWriterCaches().synchronous().getIfPresent(NamespaceName.get(NAMESPACE4))); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java index 9082a9caafcb4..361945893aed5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java @@ -255,7 +255,7 @@ public void testSkipRedeliverTemporally() { redeliverEntries.add(EntryImpl.create(1, 1, createMessage("message1", 1, "key1"))); final List readEntries = new ArrayList<>(); readEntries.add(EntryImpl.create(1, 2, createMessage("message2", 2, "key1"))); - readEntries.add(EntryImpl.create(1, 3, createMessage("message3", 3, "key2"))); + readEntries.add(EntryImpl.create(1, 3, createMessage("message3", 3, "key22"))); try { Field totalAvailablePermitsField = PersistentDispatcherMultipleConsumers.class.getDeclaredField("totalAvailablePermits"); @@ -351,7 +351,7 @@ public void testMessageRedelivery() throws Exception { // Messages with key1 are routed to consumer1 and messages with key2 are routed to consumer2 final List allEntries = new ArrayList<>(); - allEntries.add(EntryImpl.create(1, 1, createMessage("message1", 1, "key2"))); + allEntries.add(EntryImpl.create(1, 1, createMessage("message1", 1, "key22"))); allEntries.add(EntryImpl.create(1, 2, createMessage("message2", 2, "key1"))); allEntries.add(EntryImpl.create(1, 3, createMessage("message3", 3, "key1"))); allEntries.forEach(entry -> ((EntryImpl) entry).retain()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java index 6f60a13fd4894..717dfc28ac884 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java @@ -295,7 +295,8 @@ public void testPersistentPartitionedTopicUnload() throws Exception { assertFalse(pulsar.getBrokerService().getTopics().containsKey(topicName)); pulsar.getBrokerService().getTopicIfExists(topicName).get(); - assertTrue(pulsar.getBrokerService().getTopics().containsKey(topicName)); + // The map topics should only contain partitions, does not contain partitioned topic. + assertFalse(pulsar.getBrokerService().getTopics().containsKey(topicName)); // ref of partitioned-topic name should be empty assertFalse(pulsar.getBrokerService().getTopicReference(topicName).isPresent()); @@ -634,4 +635,30 @@ public void testCheckPersistencePolicies() throws Exception { assertEquals(persistentTopic.getManagedLedger().getConfig().getRetentionSizeInMB(), 1L); assertEquals(persistentTopic.getManagedLedger().getConfig().getRetentionTimeMillis(), TimeUnit.MINUTES.toMillis(1)); } + + @Test + public void testDynamicConfigurationAutoSkipNonRecoverableData() throws Exception { + pulsar.getConfiguration().setAutoSkipNonRecoverableData(false); + final String topicName = "persistent://prop/ns-abc/testAutoSkipNonRecoverableData"; + final String subName = "test_sub"; + + Consumer subscribe = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe(); + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).join().get(); + PersistentSubscription subscription = persistentTopic.getSubscription(subName); + + assertFalse(persistentTopic.ledger.getConfig().isAutoSkipNonRecoverableData()); + assertFalse(subscription.getExpiryMonitor().isAutoSkipNonRecoverableData()); + + String key = "autoSkipNonRecoverableData"; + admin.brokers().updateDynamicConfiguration(key, "true"); + Awaitility.await() + .untilAsserted(() -> assertEquals(admin.brokers().getAllDynamicConfigurations().get(key), "true")); + + assertTrue(persistentTopic.ledger.getConfig().isAutoSkipNonRecoverableData()); + assertTrue(subscription.getExpiryMonitor().isAutoSkipNonRecoverableData()); + + subscribe.close(); + admin.topics().delete(topicName); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java index 416d7ed02708e..a2401ebe19a06 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java @@ -165,7 +165,7 @@ public void testProduceAndConsumeUnderSystemNamespace() throws Exception { @Test public void testHealthCheckTopicNotOffload() throws Exception { - NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getLookupServiceAddress(), + NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getBrokerId(), pulsar.getConfig()); TopicName topicName = TopicName.get("persistent", namespaceName, BrokersBase.HEALTH_CHECK_TOPIC_SUFFIX); PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService() @@ -185,7 +185,7 @@ public void testHealthCheckTopicNotOffload() throws Exception { @Test public void testSystemNamespaceNotCreateChangeEventsTopic() throws Exception { admin.brokers().healthcheck(TopicVersion.V2); - NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getLookupServiceAddress(), + NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getBrokerId(), pulsar.getConfig()); TopicName topicName = TopicName.get("persistent", namespaceName, SystemTopicNames.NAMESPACE_EVENTS_LOCAL_NAME); Optional optionalTopic = pulsar.getBrokerService() @@ -203,7 +203,7 @@ public void testSystemNamespaceNotCreateChangeEventsTopic() throws Exception { @Test public void testHeartbeatTopicNotAllowedToSendEvent() throws Exception { admin.brokers().healthcheck(TopicVersion.V2); - NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getLookupServiceAddress(), + NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getBrokerId(), pulsar.getConfig()); TopicName topicName = TopicName.get("persistent", namespaceName, SystemTopicNames.NAMESPACE_EVENTS_LOCAL_NAME); for (int partition = 0; partition < PARTITIONS; partition ++) { @@ -218,7 +218,7 @@ public void testHeartbeatTopicNotAllowedToSendEvent() throws Exception { @Test public void testHeartbeatTopicBeDeleted() throws Exception { admin.brokers().healthcheck(TopicVersion.V2); - NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getLookupServiceAddress(), + NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getBrokerId(), pulsar.getConfig()); TopicName heartbeatTopicName = TopicName.get("persistent", namespaceName, BrokersBase.HEALTH_CHECK_TOPIC_SUFFIX); @@ -230,11 +230,11 @@ public void testHeartbeatTopicBeDeleted() throws Exception { topics = getPulsar().getNamespaceService().getListOfPersistentTopics(namespaceName).join(); Assert.assertEquals(topics.size(), 0); } - + @Test public void testHeartbeatNamespaceNotCreateTransactionInternalTopic() throws Exception { admin.brokers().healthcheck(TopicVersion.V2); - NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getLookupServiceAddress(), + NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getBrokerId(), pulsar.getConfig()); TopicName topicName = TopicName.get("persistent", namespaceName, SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java index 2611f0d89694c..4b5af5e595cdc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java @@ -54,6 +54,7 @@ import org.apache.pulsar.broker.service.ServerCnx; import org.apache.pulsar.broker.storage.ManagedLedgerStorage; import org.apache.pulsar.common.util.GracefulExecutorServicesShutdown; +import org.apache.pulsar.common.util.PortManager; import org.apache.pulsar.compaction.CompactionServiceFactory; import org.apache.pulsar.compaction.Compactor; import org.apache.pulsar.compaction.PulsarCompactionServiceFactory; @@ -157,6 +158,8 @@ public class PulsarTestContext implements AutoCloseable { private final boolean startable; + private final boolean preallocatePorts; + public ManagedLedgerFactory getManagedLedgerFactory() { return managedLedgerClientFactory.getManagedLedgerFactory(); @@ -228,7 +231,9 @@ public static class Builder { protected SpyConfig.Builder spyConfigBuilder = SpyConfig.builder(SpyConfig.SpyType.NONE); protected Consumer pulsarServiceCustomizer; protected ServiceConfiguration svcConfig = initializeConfig(); - protected Consumer configOverrideCustomizer = this::defaultOverrideServiceConfiguration; + protected Consumer configOverrideCustomizer; + + protected boolean configOverrideCalled = false; protected Function brokerServiceCustomizer = Function.identity(); protected PulsarTestContext otherContextToClose; @@ -354,6 +359,7 @@ public Builder configCustomizer(Consumer configCustomerize */ public Builder configOverride(Consumer configOverrideCustomizer) { this.configOverrideCustomizer = configOverrideCustomizer; + this.configOverrideCalled = true; return this; } @@ -538,6 +544,12 @@ public final PulsarTestContext build() { if (super.config == null) { config(svcConfig); } + handlePreallocatePorts(super.config); + if (configOverrideCustomizer != null || !configOverrideCalled) { + // call defaultOverrideServiceConfiguration if configOverrideCustomizer + // isn't explicitly set to null with `.configOverride(null)` call + defaultOverrideServiceConfiguration(super.config); + } if (configOverrideCustomizer != null) { configOverrideCustomizer.accept(super.config); } @@ -562,6 +574,37 @@ public final PulsarTestContext build() { return super.build(); } + protected void handlePreallocatePorts(ServiceConfiguration config) { + if (super.preallocatePorts) { + config.getBrokerServicePort().ifPresent(portNumber -> { + if (portNumber == 0) { + config.setBrokerServicePort(Optional.of(PortManager.nextLockedFreePort())); + } + }); + config.getBrokerServicePortTls().ifPresent(portNumber -> { + if (portNumber == 0) { + config.setBrokerServicePortTls(Optional.of(PortManager.nextLockedFreePort())); + } + }); + config.getWebServicePort().ifPresent(portNumber -> { + if (portNumber == 0) { + config.setWebServicePort(Optional.of(PortManager.nextLockedFreePort())); + } + }); + config.getWebServicePortTls().ifPresent(portNumber -> { + if (portNumber == 0) { + config.setWebServicePortTls(Optional.of(PortManager.nextLockedFreePort())); + } + }); + registerCloseable(() -> { + config.getBrokerServicePort().ifPresent(PortManager::releaseLockedPort); + config.getBrokerServicePortTls().ifPresent(PortManager::releaseLockedPort); + config.getWebServicePort().ifPresent(PortManager::releaseLockedPort); + config.getWebServicePortTls().ifPresent(PortManager::releaseLockedPort); + }); + } + } + private void initializeCommonPulsarServices(SpyConfig spyConfig) { if (super.bookKeeperClient == null && super.managedLedgerClientFactory == null) { if (super.executor == null) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index eba7f1e8c73c3..dfb19363b84fd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -1893,6 +1893,7 @@ public void testReadCommittedWithCompaction() throws Exception{ .topic(topic) .subscriptionName("sub") .subscriptionType(SubscriptionType.Exclusive) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) .readCompacted(true) .subscribe(); List result = new ArrayList<>(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TopicTransactionBufferTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TopicTransactionBufferTest.java index e5ad910cb1f10..fad785cc882ff 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TopicTransactionBufferTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TopicTransactionBufferTest.java @@ -40,8 +40,10 @@ import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferState; import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.TopicMessageId; import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.impl.MessageIdImpl; @@ -304,6 +306,62 @@ public void testGetLastMessageIdsWithOngoingTransactions() throws Exception { assertMessageId(consumer, expectedLastMessageID2, 2); } + /** + * produce 3 messages and then trigger a ledger switch, + * then create a transaction and send a transactional message. + * As there are messages in the new ledger, the reader should be able to read the messages. + * But reader.hasMessageAvailable() returns false if the entry id of max read position is -1. + * @throws Exception + */ + @Test + public void testGetLastMessageIdsWithOpenTransactionAtLedgerHead() throws Exception { + String topic = "persistent://" + NAMESPACE1 + "/testGetLastMessageIdsWithOpenTransactionAtLedgerHead"; + String subName = "my-subscription"; + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topic) + .create(); + Consumer consumer = pulsarClient.newConsumer() + .topic(topic) + .subscriptionName(subName) + .subscribe(); + MessageId expectedLastMessageID = null; + for (int i = 0; i < 3; i++) { + expectedLastMessageID = producer.newMessage().value(String.valueOf(i).getBytes()).send(); + System.out.println("expectedLastMessageID: " + expectedLastMessageID); + } + triggerLedgerSwitch(topic); + Transaction txn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.HOURS) + .build() + .get(); + producer.newMessage(txn).send(); + + Reader reader = pulsarClient.newReader() + .topic(topic) + .startMessageId(MessageId.earliest) + .create(); + assertTrue(reader.hasMessageAvailable()); + } + + private void triggerLedgerSwitch(String topicName) throws Exception{ + admin.topics().unload(topicName); + Awaitility.await().until(() -> { + CompletableFuture> topicFuture = + getPulsarServiceList().get(0).getBrokerService().getTopic(topicName, false); + if (!topicFuture.isDone() || topicFuture.isCompletedExceptionally()){ + return false; + } + Optional topicOptional = topicFuture.join(); + if (!topicOptional.isPresent()){ + return false; + } + PersistentTopic persistentTopic = (PersistentTopic) topicOptional.get(); + ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + return managedLedger.getState() == ManagedLedgerImpl.State.LedgerOpened; + }); + } + private void assertMessageId(Consumer consumer, MessageIdImpl expected, int entryOffset) throws Exception { TopicMessageIdImpl actual = (TopicMessageIdImpl) consumer.getLastMessageIds().get(0); assertEquals(expected.getEntryId(), actual.getEntryId() - entryOffset); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java index 864b481b72a8a..81ff95be3bb54 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java @@ -213,6 +213,8 @@ public void testAbortOnSubscription() throws ExecutionException, InterruptedExce @Test public void testTransactionBufferMetrics() throws Exception { + this.cleanup(); + this.setup(); //Test commit for (int i = 0; i < partitions; i++) { String topic = partitionedTopicName.getPartition(i).toString(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java index b3bd2ec466057..2fdfd3a524750 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java @@ -217,14 +217,14 @@ public void testSyncNormalPositionWhenTBRecover(boolean clientEnableTransaction, position = topicTransactionBuffer.getMaxReadPosition(); assertEquals(position, PositionImpl.EARLIEST); + // change to None state can recover + field.set(topicTransactionBuffer, TopicTransactionBufferState.State.None); + // invoke recover Method method = TopicTransactionBuffer.class.getDeclaredMethod("recover"); method.setAccessible(true); method.invoke(topicTransactionBuffer); - // change to None state can recover - field.set(topicTransactionBuffer, TopicTransactionBufferState.State.None); - // recover success again checkTopicTransactionBufferState(clientEnableTransaction, topicTransactionBuffer); @@ -236,13 +236,15 @@ public void testSyncNormalPositionWhenTBRecover(boolean clientEnableTransaction, private void checkTopicTransactionBufferState(boolean clientEnableTransaction, TopicTransactionBuffer topicTransactionBuffer) { // recover success - Awaitility.await().until(() -> { + Awaitility.await().untilAsserted(() -> { if (clientEnableTransaction) { // recover success, client enable transaction will change to Ready State - return topicTransactionBuffer.getStats(false, false).state.equals(Ready.name()); + assertEquals(topicTransactionBuffer.getStats(false, false).state, + Ready.name()); } else { // recover success, client disable transaction will change to NoSnapshot State - return topicTransactionBuffer.getStats(false, false).state.equals(NoSnapshot.name()); + assertEquals(topicTransactionBuffer.getStats(false, false).state, + NoSnapshot.name()); } }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java index a632608bf706a..0a4c5b7a318b3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java @@ -71,6 +71,7 @@ import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl; import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerWrapper; import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceUnit; +import org.apache.pulsar.broker.namespace.NamespaceEphemeralData; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.namespace.OwnedBundle; import org.apache.pulsar.broker.namespace.OwnershipCache; @@ -172,7 +173,7 @@ public void testMultipleBrokerLookup() throws Exception { // mock: return Broker2 as a Least-loaded broker when leader receives request [3] doReturn(true).when(loadManager1).isCentralized(); - SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getSafeWebServiceAddress(), null); + SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getBrokerId(), null); doReturn(Optional.of(resourceUnit)).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class)); doReturn(Optional.of(resourceUnit)).when(loadManager2).getLeastLoaded(any(ServiceUnitId.class)); loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager1)); @@ -305,7 +306,7 @@ public void testMultipleBrokerDifferentClusterLookup() throws Exception { // mock: return Broker2 as a Least-loaded broker when leader receives request doReturn(true).when(loadManager2).isCentralized(); - SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getSafeWebServiceAddress(), null); + SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getBrokerId(), null); doReturn(Optional.of(resourceUnit)).when(loadManager2).getLeastLoaded(any(ServiceUnitId.class)); loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager2)); /**** started broker-2 ****/ @@ -485,7 +486,7 @@ public void testWebserviceServiceTls() throws Exception { // request [3] doReturn(true).when(loadManager1).isCentralized(); doReturn(true).when(loadManager2).isCentralized(); - SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getWebServiceAddressTls(), null); + SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getBrokerId(), null); doReturn(Optional.of(resourceUnit)).when(loadManager2).getLeastLoaded(any(ServiceUnitId.class)); doReturn(Optional.of(resourceUnit)).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class)); @@ -579,7 +580,7 @@ public void testSplitUnloadLookupTest() throws Exception { loadManagerField.set(pulsar2.getNamespaceService(), new AtomicReference<>(loadManager2)); // mock: return Broker1 as a Least-loaded broker when leader receives request [3] doReturn(true).when(loadManager1).isCentralized(); - SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getSafeWebServiceAddress(), null); + SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getBrokerId(), null); doReturn(Optional.of(resourceUnit)).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class)); doReturn(Optional.of(resourceUnit)).when(loadManager2).getLeastLoaded(any(ServiceUnitId.class)); loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager1)); @@ -694,7 +695,7 @@ public void testModularLoadManagerSplitBundle() throws Exception { loadManagerField.set(pulsar2.getNamespaceService(), new AtomicReference<>(loadManager2)); // mock: return Broker1 as a Least-loaded broker when leader receives request [3] doReturn(true).when(loadManager1).isCentralized(); - SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getSafeWebServiceAddress(), null); + SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getBrokerId(), null); Optional res = Optional.of(resourceUnit); doReturn(res).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class)); doReturn(res).when(loadManager2).getLeastLoaded(any(ServiceUnitId.class)); @@ -760,7 +761,7 @@ public void testModularLoadManagerSplitBundle() throws Exception { }); // Unload the NamespacePolicies and AntiAffinity check. - String currentBroker = String.format("%s:%d", "localhost", pulsar.getListenPortHTTP().get()); + String currentBroker = pulsar.getBrokerId(); assertTrue(loadManager.shouldNamespacePoliciesUnload(namespace,"0x00000000_0xffffffff", currentBroker)); assertTrue(loadManager.shouldAntiAffinityNamespaceUnload(namespace,"0x00000000_0xffffffff", currentBroker)); @@ -1208,4 +1209,42 @@ private void makeAcquireBundleLockSuccess() throws Exception { mockZooKeeper.unsetAlwaysFail(); } } + + @Test(timeOut = 30000) + public void testLookupConnectionNotCloseIfFailedToAcquireOwnershipOfBundle() throws Exception { + String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createNonPartitionedTopic(tpName); + final var pulsarClientImpl = (PulsarClientImpl) pulsarClient; + final var cache = pulsar.getNamespaceService().getOwnershipCache(); + final var bundle = pulsar.getNamespaceService().getBundle(TopicName.get(tpName)); + final var value = cache.getOwnerAsync(bundle).get().orElse(null); + assertNotNull(value); + + cache.invalidateLocalOwnerCache(); + final var lock = pulsar.getCoordinationService().getLockManager(NamespaceEphemeralData.class) + .acquireLock(ServiceUnitUtils.path(bundle), new NamespaceEphemeralData()).join(); + lock.updateValue(null); + log.info("Updated bundle {} with null", bundle.getBundleRange()); + + // wait for the system topic reader to __change_events is closed, otherwise the test will be affected + Thread.sleep(500); + + final var future = pulsarClientImpl.getLookup().getBroker(TopicName.get(tpName)); + final var cnx = pulsarClientImpl.getCnxPool().getConnections().stream().findAny() + .map(CompletableFuture::join).orElse(null); + assertNotNull(cnx); + + try { + future.get(); + fail(); + } catch (ExecutionException e) { + log.info("getBroker failed with {}: {}", e.getCause().getClass().getName(), e.getMessage()); + assertTrue(e.getCause() instanceof PulsarClientException.BrokerMetadataException); + assertTrue(cnx.ctx().channel().isActive()); + lock.updateValue(value); + lock.release(); + assertTrue(e.getMessage().contains("Failed to acquire ownership")); + pulsarClientImpl.getLookup().getBroker(TopicName.get(tpName)).get(); + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java index ea93ece549e27..4433670c7a595 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java @@ -140,7 +140,8 @@ public void testDeadLetterTopicWithMessageKey() throws Exception { public void testDeadLetterTopicWithProducerName() throws Exception { final String topic = "persistent://my-property/my-ns/dead-letter-topic"; final String subscription = "my-subscription"; - String deadLetterProducerName = String.format("%s-%s-DLQ", topic, subscription); + final String consumerName = "my-consumer"; + String deadLetterProducerName = String.format("%s-%s-%s-DLQ", topic, subscription, consumerName); final int maxRedeliveryCount = 1; @@ -149,6 +150,7 @@ public void testDeadLetterTopicWithProducerName() throws Exception { Consumer consumer = pulsarClient.newConsumer(Schema.BYTES) .topic(topic) .subscriptionName(subscription) + .consumerName(consumerName) .subscriptionType(SubscriptionType.Shared) .ackTimeout(1, TimeUnit.SECONDS) .deadLetterPolicy(DeadLetterPolicy.builder().maxRedeliverCount(maxRedeliveryCount).build()) @@ -929,6 +931,9 @@ public void testDeadLetterTopicWithInitialSubscriptionAndMultiConsumers() throws assertTrue(admin.topics().getSubscriptions(deadLetterTopic).contains(dlqInitialSub)); }); + // We should assert that all consumers are able to produce messages to DLQ + assertEquals(admin.topics().getStats(deadLetterTopic).getPublishers().size(), 2); + Consumer deadLetterConsumer = newPulsarClient.newConsumer(Schema.BYTES) .topic(deadLetterTopic) .subscriptionName(dlqInitialSub) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java index fc103a46027c0..bd0119823fd95 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java @@ -692,8 +692,8 @@ public void testBlockBrokerDispatching() { try { final int waitMills = 500; final int maxUnAckPerBroker = 200; - final double unAckMsgPercentagePerDispatcher = 10; - int maxUnAckPerDispatcher = (int) ((maxUnAckPerBroker * unAckMsgPercentagePerDispatcher) / 100); // 200 * + final double unAckMsgPercentagePerDispatcher = 0.1; + int maxUnAckPerDispatcher = (int) (maxUnAckPerBroker * unAckMsgPercentagePerDispatcher); // 200 * // 10% = 20 // messages pulsar.getConfiguration().setMaxUnackedMessagesPerBroker(maxUnAckPerBroker); @@ -907,8 +907,8 @@ public void testBrokerDispatchBlockAndSubAckBackRequiredMsgs() { .getMaxUnackedMessagesPerSubscriptionOnBrokerBlocked(); try { final int maxUnAckPerBroker = 200; - final double unAckMsgPercentagePerDispatcher = 10; - int maxUnAckPerDispatcher = (int) ((maxUnAckPerBroker * unAckMsgPercentagePerDispatcher) / 100); // 200 * + final double unAckMsgPercentagePerDispatcher = 0.1; + int maxUnAckPerDispatcher = (int) (maxUnAckPerBroker * unAckMsgPercentagePerDispatcher); // 200 * // 10% = 20 // messages pulsar.getConfiguration().setMaxUnackedMessagesPerBroker(maxUnAckPerBroker); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java index 451f93067b2ca..7707abafde8de 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java @@ -27,6 +27,7 @@ import com.google.common.collect.Lists; +import java.time.Duration; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -35,16 +36,21 @@ import java.util.stream.IntStream; import io.netty.util.Timeout; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.InjectedClientCnxClientBuilder; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.RegexSubscriptionMode; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.common.api.proto.BaseCommand; +import org.apache.pulsar.common.api.proto.CommandWatchTopicListSuccess; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.awaitility.Awaitility; @@ -53,6 +59,7 @@ import org.slf4j.LoggerFactory; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @Test(groups = "broker-impl") @@ -620,13 +627,28 @@ public void testStartEmptyPatternConsumer() throws Exception { producer3.close(); } - @Test(timeOut = testTimeout) - public void testAutoSubscribePatterConsumerFromBrokerWatcher() throws Exception { - String key = "AutoSubscribePatternConsumer"; - String subscriptionName = "my-ex-subscription-" + key; + @DataProvider(name= "delayTypesOfWatchingTopics") + public Object[][] delayTypesOfWatchingTopics(){ + return new Object[][]{ + {true}, + {false} + }; + } - Pattern pattern = Pattern.compile("persistent://my-property/my-ns/pattern-topic.*"); - Consumer consumer = pulsarClient.newConsumer() + @Test(timeOut = testTimeout, dataProvider = "delayTypesOfWatchingTopics") + public void testAutoSubscribePatterConsumerFromBrokerWatcher(boolean delayWatchingTopics) throws Exception { + final String key = "AutoSubscribePatternConsumer"; + final String subscriptionName = "my-ex-subscription-" + key; + final Pattern pattern = Pattern.compile("persistent://my-property/my-ns/pattern-topic.*"); + + PulsarClient client = null; + if (delayWatchingTopics) { + client = createDelayWatchTopicsClient(); + } else { + client = pulsarClient; + } + + Consumer consumer = client.newConsumer() .topicsPattern(pattern) // Disable automatic discovery. .patternAutoDiscoveryPeriod(1000) @@ -636,12 +658,6 @@ public void testAutoSubscribePatterConsumerFromBrokerWatcher() throws Exception .receiverQueueSize(4) .subscribe(); - // Wait topic list watcher creation. - Awaitility.await().untilAsserted(() -> { - CompletableFuture completableFuture = WhiteboxImpl.getInternalState(consumer, "watcherFuture"); - assertTrue(completableFuture.isDone() && !completableFuture.isCompletedExceptionally()); - }); - // 1. create partition String topicName = "persistent://my-property/my-ns/pattern-topic-1-" + key; TenantInfoImpl tenantInfo = createDefaultTenantInfo(); @@ -657,7 +673,162 @@ public void testAutoSubscribePatterConsumerFromBrokerWatcher() throws Exception assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitionedTopics().size(), 1); }); + // cleanup. + consumer.close(); + admin.topics().deletePartitionedTopic(topicName); + if (delayWatchingTopics) { + client.close(); + } + } + + @DataProvider(name= "regexpConsumerArgs") + public Object[][] regexpConsumerArgs(){ + return new Object[][]{ + {true, true}, + {true, false}, + {false, true}, + {false, false} + }; + } + + private void waitForTopicListWatcherStarted(Consumer consumer) { + Awaitility.await().untilAsserted(() -> { + CompletableFuture completableFuture = WhiteboxImpl.getInternalState(consumer, "watcherFuture"); + log.info("isDone: {}, isCompletedExceptionally: {}", completableFuture.isDone(), + completableFuture.isCompletedExceptionally()); + assertTrue(completableFuture.isDone() && !completableFuture.isCompletedExceptionally()); + }); + } + + @Test(timeOut = testTimeout, dataProvider = "regexpConsumerArgs") + public void testPreciseRegexpSubscribe(boolean partitioned, boolean createTopicAfterWatcherStarted) throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final String subscriptionName = "s1"; + final Pattern pattern = Pattern.compile(String.format("%s$", topicName)); + + Consumer consumer = pulsarClient.newConsumer() + .topicsPattern(pattern) + // Disable automatic discovery. + .patternAutoDiscoveryPeriod(1000) + .subscriptionName(subscriptionName) + .subscriptionType(SubscriptionType.Shared) + .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS) + .receiverQueueSize(4) + .subscribe(); + if (createTopicAfterWatcherStarted) { + waitForTopicListWatcherStarted(consumer); + } + + // 1. create topic. + if (partitioned) { + admin.topics().createPartitionedTopic(topicName, 1); + } else { + admin.topics().createNonPartitionedTopic(topicName); + } + + // 2. verify consumer can subscribe the topic. + assertSame(pattern, ((PatternMultiTopicsConsumerImpl) consumer).getPattern()); + Awaitility.await().untilAsserted(() -> { + assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitions().size(), 1); + assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getConsumers().size(), 1); + if (partitioned) { + assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitionedTopics().size(), 1); + } else { + assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitionedTopics().size(), 0); + } + }); + + // cleanup. consumer.close(); + if (partitioned) { + admin.topics().deletePartitionedTopic(topicName); + } else { + admin.topics().delete(topicName); + } + } + + @DataProvider(name= "partitioned") + public Object[][] partitioned(){ + return new Object[][]{ + {true}, + {true} + }; + } + + @Test(timeOut = 240 * 1000, dataProvider = "partitioned") + public void testPreciseRegexpSubscribeDisabledTopicWatcher(boolean partitioned) throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final String subscriptionName = "s1"; + final Pattern pattern = Pattern.compile(String.format("%s$", topicName)); + + // Close all ServerCnx by close client-side sockets to make the config changes effect. + pulsar.getConfig().setEnableBrokerSideSubscriptionPatternEvaluation(false); + reconnectAllConnections(); + + Consumer consumer = pulsarClient.newConsumer() + .topicsPattern(pattern) + // Disable brokerSideSubscriptionPatternEvaluation will leading disable topic list watcher. + // So set patternAutoDiscoveryPeriod to a little value. + .patternAutoDiscoveryPeriod(1) + .subscriptionName(subscriptionName) + .subscriptionType(SubscriptionType.Shared) + .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS) + .receiverQueueSize(4) + .subscribe(); + + // 1. create topic. + if (partitioned) { + admin.topics().createPartitionedTopic(topicName, 1); + } else { + admin.topics().createNonPartitionedTopic(topicName); + } + + // 2. verify consumer can subscribe the topic. + // Since the minimum value of `patternAutoDiscoveryPeriod` is 60s, we set the test timeout to a triple value. + assertSame(pattern, ((PatternMultiTopicsConsumerImpl) consumer).getPattern()); + Awaitility.await().atMost(Duration.ofMinutes(3)).untilAsserted(() -> { + assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitions().size(), 1); + assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getConsumers().size(), 1); + if (partitioned) { + assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitionedTopics().size(), 1); + } else { + assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitionedTopics().size(), 0); + } + }); + + // cleanup. + consumer.close(); + if (partitioned) { + admin.topics().deletePartitionedTopic(topicName); + } else { + admin.topics().delete(topicName); + } + // Close all ServerCnx by close client-side sockets to make the config changes effect. + pulsar.getConfig().setEnableBrokerSideSubscriptionPatternEvaluation(true); + reconnectAllConnections(); + } + + private PulsarClient createDelayWatchTopicsClient() throws Exception { + ClientBuilderImpl clientBuilder = (ClientBuilderImpl) PulsarClient.builder().serviceUrl(lookupUrl.toString()); + return InjectedClientCnxClientBuilder.create(clientBuilder, + (conf, eventLoopGroup) -> new ClientCnx(conf, eventLoopGroup) { + public CompletableFuture newWatchTopicList( + BaseCommand command, long requestId) { + // Inject 2 seconds delay when sending command New Watch Topics. + CompletableFuture res = new CompletableFuture<>(); + new Thread(() -> { + sleepSeconds(2); + super.newWatchTopicList(command, requestId).whenComplete((v, ex) -> { + if (ex != null) { + res.completeExceptionally(ex); + } else { + res.complete(v); + } + }); + }).start(); + return res; + } + }); } // simulate subscribe a pattern which has 3 topics, but then matched topic added in. diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerConsumerInternalTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerConsumerInternalTest.java index f05f735635746..240d8d2304768 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerConsumerInternalTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerConsumerInternalTest.java @@ -20,22 +20,32 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; + +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.service.ServerCnx; +import org.apache.pulsar.client.api.BatcherBuilder; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerConsumerBase; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.common.api.proto.CommandCloseProducer; import org.awaitility.Awaitility; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; /** * Different with {@link org.apache.pulsar.client.api.SimpleProducerConsumerTest}, this class can visit the variables - * of {@link ConsumerImpl} which are modified `protected`. + * of {@link ConsumerImpl} or {@link ProducerImpl} which have protected or default access modifiers. */ -@Test(groups = "broker-api") +@Slf4j +@Test(groups = "broker-impl") public class ProducerConsumerInternalTest extends ProducerConsumerBase { @BeforeClass(alwaysRun = true) @@ -144,4 +154,36 @@ public void testExclusiveConsumerWillAlwaysRetryEvenIfReceivedConsumerBusyError( consumer.close(); admin.topics().delete(topicName, false); } + + @DataProvider(name = "containerBuilder") + public Object[][] containerBuilderProvider() { + return new Object[][] { + { BatcherBuilder.DEFAULT }, + { BatcherBuilder.KEY_BASED } + }; + } + + @Test(timeOut = 30000, dataProvider = "containerBuilder") + public void testSendTimerCheckForBatchContainer(BatcherBuilder batcherBuilder) throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://my-property/my-ns/tp_"); + @Cleanup Producer producer = pulsarClient.newProducer().topic(topicName) + .batcherBuilder(batcherBuilder) + .sendTimeout(1, TimeUnit.SECONDS) + .batchingMaxPublishDelay(100, TimeUnit.MILLISECONDS) + .batchingMaxMessages(1000) + .create(); + + log.info("Before sendAsync msg-0: {}", System.nanoTime()); + CompletableFuture future = producer.sendAsync("msg-0".getBytes()); + future.thenAccept(msgId -> log.info("msg-0 done: {} (msgId: {})", System.nanoTime(), msgId)); + future.get(); // t: the current time point + + ((ProducerImpl) producer).triggerSendTimer(); // t+1000ms && t+2000ms: run() will be called again + + Thread.sleep(1950); // t+2050ms: the batch timer is expired, which happens after run() is called + log.info("Before sendAsync msg-1: {}", System.nanoTime()); + future = producer.sendAsync("msg-1".getBytes()); + future.thenAccept(msgId -> log.info("msg-1 done: {} (msgId: {})", System.nanoTime(), msgId)); + future.get(); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ReaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ReaderTest.java index 64a5da43d4485..4e4dc8273d3f0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ReaderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ReaderTest.java @@ -785,4 +785,32 @@ public void testReaderListenerAcknowledgement() admin.topics().deletePartitionedTopic(partitionedTopic); } + @Test + public void testReaderReconnectedFromNextEntry() throws Exception { + final String topic = "persistent://my-property/my-ns/testReaderReconnectedFromNextEntry"; + Reader reader = pulsarClient.newReader(Schema.STRING).topic(topic).receiverQueueSize(1) + .startMessageId(MessageId.earliest).create(); + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + + // Send 3 and consume 1. + producer.send("1"); + producer.send("2"); + producer.send("3"); + Message msg1 = reader.readNext(2, TimeUnit.SECONDS); + assertEquals(msg1.getValue(), "1"); + + // Trigger reader reconnect. + admin.topics().unload(topic); + + // For non-durable we are going to restart from the next entry. + Message msg2 = reader.readNext(2, TimeUnit.SECONDS); + assertEquals(msg2.getValue(), "2"); + Message msg3 = reader.readNext(2, TimeUnit.SECONDS); + assertEquals(msg3.getValue(), "3"); + + // cleanup. + reader.close(); + producer.close(); + admin.topics().delete(topic, false); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java index 51b32c2b44ecf..c343ab0d6e294 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java @@ -34,6 +34,7 @@ import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageIdAdv; +import org.apache.pulsar.client.api.MessageListener; import org.apache.pulsar.client.api.MessageRouter; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; @@ -57,22 +58,27 @@ import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; - import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; import java.util.Set; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.TreeSet; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -1394,4 +1400,76 @@ public Map getActiveConsumers() { } } + @DataProvider + public static Object[][] seekByFunction() { + return new Object[][] { + { true }, { false } + }; + } + + @Test(timeOut = 30000, dataProvider = "seekByFunction") + public void testSeekToNewerPosition(boolean seekByFunction) throws Exception { + final var topic1 = TopicName.get(newTopicName()).toString() + .replace("my-property", "public").replace("my-ns", "default"); + final var topic2 = TopicName.get(newTopicName()).toString() + .replace("my-property", "public").replace("my-ns", "default"); + @Cleanup final var producer1 = pulsarClient.newProducer(Schema.STRING).topic(topic1).create(); + @Cleanup final var producer2 = pulsarClient.newProducer(Schema.STRING).topic(topic2).create(); + producer1.send("1-0"); + producer2.send("2-0"); + producer1.send("1-1"); + producer2.send("2-1"); + final var consumer1 = pulsarClient.newConsumer(Schema.STRING) + .topics(Arrays.asList(topic1, topic2)).subscriptionName("sub") + .ackTimeout(1, TimeUnit.SECONDS) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe(); + final var timestamps = new ArrayList(); + for (int i = 0; i < 4; i++) { + timestamps.add(consumer1.receive().getPublishTime()); + } + timestamps.sort(Comparator.naturalOrder()); + final var timestamp = timestamps.get(2); + consumer1.close(); + + final Function, CompletableFuture> seekAsync = consumer -> { + final var future = seekByFunction ? consumer.seekAsync(__ -> timestamp) : consumer.seekAsync(timestamp); + assertEquals(((ConsumerBase) consumer).getIncomingMessageSize(), 0L); + assertEquals(((ConsumerBase) consumer).getTotalIncomingMessages(), 0); + assertTrue(((ConsumerBase) consumer).getUnAckedMessageTracker().isEmpty()); + return future; + }; + + @Cleanup final var consumer2 = pulsarClient.newConsumer(Schema.STRING) + .topics(Arrays.asList(topic1, topic2)).subscriptionName("sub-2") + .ackTimeout(1, TimeUnit.SECONDS) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe(); + seekAsync.apply(consumer2).get(); + final var values = new TreeSet(); + for (int i = 0; i < 2; i++) { + values.add(consumer2.receive().getValue()); + } + assertEquals(values, new TreeSet<>(Arrays.asList("1-1", "2-1"))); + + final var valuesInListener = new CopyOnWriteArrayList(); + @Cleanup final var consumer3 = pulsarClient.newConsumer(Schema.STRING) + .topics(Arrays.asList(topic1, topic2)).subscriptionName("sub-3") + .messageListener((MessageListener) (__, msg) -> valuesInListener.add(msg.getValue())) + .ackTimeout(1, TimeUnit.SECONDS) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe(); + seekAsync.apply(consumer3).get(); + if (valuesInListener.isEmpty()) { + Awaitility.await().untilAsserted(() -> assertEquals(valuesInListener.size(), 2)); + assertEquals(valuesInListener.stream().sorted().toList(), Arrays.asList("1-1", "2-1")); + } // else: consumer3 has passed messages to the listener before seek, in this case we cannot assume anything + + @Cleanup final var consumer4 = pulsarClient.newConsumer(Schema.STRING) + .topics(Arrays.asList(topic1, topic2)).subscriptionName("sub-4") + .ackTimeout(1, TimeUnit.SECONDS) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe(); + seekAsync.apply(consumer4).get(); + final var valuesInReceiveAsync = new ArrayList(); + valuesInReceiveAsync.add(consumer4.receiveAsync().get().getValue()); + valuesInReceiveAsync.add(consumer4.receiveAsync().get().getValue()); + assertEquals(valuesInReceiveAsync.stream().sorted().toList(), Arrays.asList("1-1", "2-1")); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionRetentionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionRetentionTest.java index 055c595fbfec8..88d923f74e196 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionRetentionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionRetentionTest.java @@ -38,6 +38,7 @@ import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; @@ -45,9 +46,13 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.naming.SystemTopicNames; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.awaitility.Awaitility; +import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -212,6 +217,51 @@ public void testCompactionRetentionOnTopicCreationWithTopicPolicies() throws Exc ); } + @Test + public void testRetentionPolicesForSystemTopic() throws Exception { + String namespace = "my-tenant/my-ns"; + String topicPrefix = "persistent://" + namespace + "/"; + admin.namespaces().setRetention(namespace, new RetentionPolicies(-1, -1)); + // Check event topics and transaction internal topics. + for (String eventTopic : SystemTopicNames.EVENTS_TOPIC_NAMES) { + checkSystemTopicRetentionPolicy(topicPrefix + eventTopic); + } + checkSystemTopicRetentionPolicy(topicPrefix + SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN); + checkSystemTopicRetentionPolicy(topicPrefix + SystemTopicNames.TRANSACTION_COORDINATOR_LOG); + checkSystemTopicRetentionPolicy(topicPrefix + SystemTopicNames.PENDING_ACK_STORE_SUFFIX); + + // Check common topics. + checkCommonTopicRetentionPolicy(topicPrefix + "my-topic" + System.nanoTime()); + // Specify retention policies for system topic. + pulsar.getConfiguration().setTopicLevelPoliciesEnabled(true); + pulsar.getConfiguration().setSystemTopicEnabled(true); + admin.topics().createNonPartitionedTopic(topicPrefix + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT); + admin.topicPolicies().setRetention(topicPrefix + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT, + new RetentionPolicies(10, 10)); + Awaitility.await().untilAsserted(() -> { + checkTopicRetentionPolicy(topicPrefix + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT, + new RetentionPolicies(10, 10)); + }); + } + + private void checkSystemTopicRetentionPolicy(String topicName) throws Exception { + checkTopicRetentionPolicy(topicName, new RetentionPolicies(0, 0)); + + } + + private void checkCommonTopicRetentionPolicy(String topicName) throws Exception { + checkTopicRetentionPolicy(topicName, new RetentionPolicies(-1, -1)); + } + + private void checkTopicRetentionPolicy(String topicName, RetentionPolicies retentionPolicies) throws Exception { + ManagedLedgerConfig config = pulsar.getBrokerService() + .getManagedLedgerConfig(TopicName.get(topicName)).get(); + Assert.assertEquals(config.getRetentionSizeInMB(), retentionPolicies.getRetentionSizeInMB()); + Assert.assertEquals(config.getRetentionTimeMillis(), retentionPolicies.getRetentionTimeInMinutes() < 0 + ? retentionPolicies.getRetentionTimeInMinutes() + : retentionPolicies.getRetentionTimeInMinutes() * 60000L); + } + private void testCompactionCursorRetention(String topic) throws Exception { Set keys = Sets.newHashSet("a", "b", "c"); Set keysToExpire = Sets.newHashSet("x1", "x2"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java index 9eda547968388..f0010096b1e52 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java @@ -25,6 +25,7 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; import com.google.common.collect.Sets; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -46,21 +47,26 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; import lombok.Cleanup; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.api.OpenBuilder; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerInfo; import org.apache.bookkeeper.mledger.Position; +import org.apache.commons.lang3.mutable.MutableLong; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.service.persistent.SystemTopic; import org.apache.pulsar.client.admin.LongRunningProcessStatus; @@ -81,6 +87,7 @@ import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.impl.BatchMessageIdImpl; import org.apache.pulsar.client.impl.ConsumerImpl; +import org.apache.pulsar.common.api.proto.CommandAck; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; @@ -1771,9 +1778,9 @@ public void testReadUnCompacted(boolean batchEnabled) throws PulsarClientExcepti @SneakyThrows @Test public void testHealthCheckTopicNotCompacted() { - NamespaceName heartbeatNamespaceV1 = NamespaceService.getHeartbeatNamespace(pulsar.getLookupServiceAddress(), pulsar.getConfiguration()); + NamespaceName heartbeatNamespaceV1 = NamespaceService.getHeartbeatNamespace(pulsar.getBrokerId(), pulsar.getConfiguration()); String topicV1 = "persistent://" + heartbeatNamespaceV1.toString() + "/healthcheck"; - NamespaceName heartbeatNamespaceV2 = NamespaceService.getHeartbeatNamespaceV2(pulsar.getLookupServiceAddress(), pulsar.getConfiguration()); + NamespaceName heartbeatNamespaceV2 = NamespaceService.getHeartbeatNamespaceV2(pulsar.getBrokerId(), pulsar.getConfiguration()); String topicV2 = heartbeatNamespaceV2.toString() + "/healthcheck"; Producer producer1 = pulsarClient.newProducer().topic(topicV1).create(); Producer producer2 = pulsarClient.newProducer().topic(topicV2).create(); @@ -1868,6 +1875,7 @@ public void testReceiverQueueSize() throws Exception { ConsumerImpl consumer = (ConsumerImpl) client.newConsumer(Schema.STRING) .topic(topicName).readCompacted(true).receiverQueueSize(receiveQueueSize).subscriptionName(subName) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) .subscribe(); //Give some time to consume @@ -1911,6 +1919,7 @@ public void testDispatcherMaxReadSizeBytes() throws Exception { ConsumerImpl consumer = (ConsumerImpl) client.newConsumer(Schema.BYTES) .topic(topicName).readCompacted(true).receiverQueueSize(receiveQueueSize).subscriptionName(subName) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) .subscribe(); Awaitility.await().untilAsserted(() -> { @@ -2012,4 +2021,347 @@ public void testCompactionDuplicate() throws Exception { } } } + + @Test + public void testDeleteCompactedLedger() throws Exception { + String topicName = "persistent://my-property/use/my-ns/testDeleteCompactedLedger"; + + final String subName = "my-sub"; + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(false).topic(topicName).create(); + + pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).readCompacted(true).subscribe().close(); + + for (int i = 0; i < 10; i++) { + producer.newMessage().key(String.valueOf(i % 2)).value(String.valueOf(i)).sendAsync(); + } + producer.flush(); + + compact(topicName); + + MutableLong compactedLedgerId = new MutableLong(-1); + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats stats = admin.topics().getInternalStats(topicName); + Assert.assertNotEquals(stats.compactedLedger.ledgerId, -1L); + compactedLedgerId.setValue(stats.compactedLedger.ledgerId); + Assert.assertEquals(stats.compactedLedger.entries, 2L); + }); + + // delete compacted ledger + admin.topics().deleteSubscription(topicName, "__compaction"); + + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats stats = admin.topics().getInternalStats(topicName); + Assert.assertEquals(stats.compactedLedger.ledgerId, -1L); + Assert.assertEquals(stats.compactedLedger.entries, -1L); + assertThrows(BKException.BKNoSuchLedgerExistsException.class, () -> pulsarTestContext.getBookKeeperClient() + .openLedger(compactedLedgerId.getValue(), BookKeeper.DigestType.CRC32C, new byte[]{})); + }); + + compact(topicName); + + MutableLong compactedLedgerId2 = new MutableLong(-1); + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats stats = admin.topics().getInternalStats(topicName); + Assert.assertNotEquals(stats.compactedLedger.ledgerId, -1L); + compactedLedgerId2.setValue(stats.compactedLedger.ledgerId); + Assert.assertEquals(stats.compactedLedger.entries, 2L); + }); + + producer.close(); + admin.topics().delete(topicName); + + Awaitility.await().untilAsserted(() -> assertThrows(BKException.BKNoSuchLedgerExistsException.class, + () -> pulsarTestContext.getBookKeeperClient().openLedger( + compactedLedgerId2.getValue(), BookKeeper.DigestType.CRC32, new byte[]{}))); + } + + @Test + public void testDeleteCompactedLedgerWithSlowAck() throws Exception { + // Disable topic level policies, since block ack thread may also block thread of delete topic policies. + conf.setTopicLevelPoliciesEnabled(false); + restartBroker(); + + String topicName = "persistent://my-property/use/my-ns/testDeleteCompactedLedgerWithSlowAck"; + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(false).topic(topicName).create(); + + pulsarClient.newConsumer().topic(topicName).subscriptionType(SubscriptionType.Exclusive) + .subscriptionName(Compactor.COMPACTION_SUBSCRIPTION) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).readCompacted(true).subscribe() + .close(); + + for (int i = 0; i < 10; i++) { + producer.newMessage().key(String.valueOf(i % 2)).value(String.valueOf(i)).sendAsync(); + } + producer.flush(); + + PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get(); + PersistentSubscription subscription = spy(topic.getSubscription(Compactor.COMPACTION_SUBSCRIPTION)); + topic.getSubscriptions().put(Compactor.COMPACTION_SUBSCRIPTION, subscription); + + AtomicLong compactedLedgerId = new AtomicLong(-1); + AtomicBoolean pauseAck = new AtomicBoolean(); + Mockito.doAnswer(invocationOnMock -> { + Map properties = (Map) invocationOnMock.getArguments()[2]; + log.info("acknowledgeMessage properties: {}", properties); + compactedLedgerId.set(properties.get(Compactor.COMPACTED_TOPIC_LEDGER_PROPERTY)); + pauseAck.set(true); + while (pauseAck.get()) { + Thread.sleep(200); + } + return invocationOnMock.callRealMethod(); + }).when(subscription).acknowledgeMessage(Mockito.any(), Mockito.eq( + CommandAck.AckType.Cumulative), Mockito.any()); + + admin.topics().triggerCompaction(topicName); + + while (!pauseAck.get()) { + Thread.sleep(100); + } + + CompletableFuture currentCompaction = + (CompletableFuture) FieldUtils.readDeclaredField(topic, "currentCompaction", true); + CompletableFuture spyCurrentCompaction = spy(currentCompaction); + FieldUtils.writeDeclaredField(topic, "currentCompaction", spyCurrentCompaction, true); + currentCompaction.whenComplete((obj, throwable) -> { + if (throwable != null) { + spyCurrentCompaction.completeExceptionally(throwable); + } else { + spyCurrentCompaction.complete(obj); + } + }); + Mockito.doAnswer(invocationOnMock -> { + pauseAck.set(false); + return invocationOnMock.callRealMethod(); + }).when(spyCurrentCompaction).handle(Mockito.any()); + + admin.topics().delete(topicName, true); + + Awaitility.await().untilAsserted(() -> assertThrows(BKException.BKNoSuchLedgerExistsException.class, + () -> pulsarTestContext.getBookKeeperClient().openLedger( + compactedLedgerId.get(), BookKeeper.DigestType.CRC32, new byte[]{}))); + } + + @Test + public void testCompactionWithTTL() throws Exception { + String topicName = "persistent://my-property/use/my-ns/testCompactionWithTTL"; + String subName = "sub"; + pulsarClient.newConsumer(Schema.STRING).topic(topicName).subscriptionName(subName).readCompacted(true) + .subscribe().close(); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(false).topic(topicName).create(); + + producer.newMessage().key("K1").value("V1").send(); + producer.newMessage().key("K2").value("V2").send(); + + admin.topics().triggerCompaction(topicName); + + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS); + }); + + producer.newMessage().key("K1").value("V3").send(); + producer.newMessage().key("K2").value("V4").send(); + + Thread.sleep(1000); + + // expire messages + admin.topics().expireMessagesForAllSubscriptions(topicName, 1); + + // trim the topic + admin.topics().unload(topicName); + + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(topicName, false); + assertEquals(internalStats.numberOfEntries, 4); + }); + + producer.newMessage().key("K3").value("V5").send(); + + admin.topics().triggerCompaction(topicName); + + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS); + }); + + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.STRING).topic(topicName) + .subscriptionName("sub-2") + .readCompacted(true) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + List result = new ArrayList<>(); + while (true) { + Message receive = consumer.receive(2, TimeUnit.SECONDS); + if (receive == null) { + break; + } + + result.add(receive.getValue()); + } + + Assert.assertEquals(result, List.of("V3", "V4", "V5")); + } + + @Test + public void testAcknowledgeWithReconnection() throws Exception { + final String topicName = "persistent://my-property/use/my-ns/testAcknowledge" + UUID.randomUUID(); + final String subName = "my-sub"; + @Cleanup + PulsarClient client = newPulsarClient(lookupUrl.toString(), 100); + Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(false).topic(topicName).create(); + + List expected = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + producer.newMessage().key(String.valueOf(i)).value(String.valueOf(i)).send(); + expected.add(String.valueOf(i)); + } + producer.flush(); + + admin.topics().triggerCompaction(topicName); + + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS); + }); + + // trim the topic + admin.topics().unload(topicName); + + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(topicName, false); + assertEquals(internalStats.numberOfEntries, 0); + }); + + ConsumerImpl consumer = (ConsumerImpl) client.newConsumer(Schema.STRING) + .topic(topicName).readCompacted(true).receiverQueueSize(1).subscriptionName(subName) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .isAckReceiptEnabled(true) + .subscribe(); + + List results = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + Message message = consumer.receive(3, TimeUnit.SECONDS); + if (message == null) { + break; + } + results.add(message.getValue()); + consumer.acknowledge(message); + } + + Awaitility.await().untilAsserted(() -> + assertEquals(admin.topics().getStats(topicName, true).getSubscriptions().get(subName).getMsgBacklog(), + 5)); + + // Make consumer reconnect to broker + admin.topics().unload(topicName); + + // Wait for consumer to reconnect and clear incomingMessages + consumer.pause(); + Awaitility.await().untilAsserted(() -> { + Assert.assertEquals(consumer.numMessagesInQueue(), 0); + }); + consumer.resume(); + + for (int i = 0; i < 5; i++) { + Message message = consumer.receive(3, TimeUnit.SECONDS); + if (message == null) { + break; + } + results.add(message.getValue()); + consumer.acknowledge(message); + } + + Awaitility.await().untilAsserted(() -> + assertEquals(admin.topics().getStats(topicName, true).getSubscriptions().get(subName).getMsgBacklog(), + 0)); + + Assert.assertEquals(results, expected); + + Message message = consumer.receive(3, TimeUnit.SECONDS); + Assert.assertNull(message); + + // Make consumer reconnect to broker + admin.topics().unload(topicName); + + producer.newMessage().key("K").value("V").send(); + Message message2 = consumer.receive(3, TimeUnit.SECONDS); + Assert.assertEquals(message2.getValue(), "V"); + consumer.acknowledge(message2); + + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(topicName); + Assert.assertEquals(internalStats.lastConfirmedEntry, + internalStats.cursors.get(subName).markDeletePosition); + }); + + consumer.close(); + producer.close(); + } + + @Test + public void testEarliestSubsAfterRollover() throws Exception { + final String topicName = "persistent://my-property/use/my-ns/testEarliestSubsAfterRollover" + UUID.randomUUID(); + final String subName = "my-sub"; + @Cleanup + PulsarClient client = newPulsarClient(lookupUrl.toString(), 100); + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(false).topic(topicName).create(); + + List expected = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + producer.newMessage().key(String.valueOf(i)).value(String.valueOf(i)).send(); + expected.add(String.valueOf(i)); + } + producer.flush(); + + admin.topics().triggerCompaction(topicName); + + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS); + }); + + // trim the topic + admin.topics().unload(topicName); + + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(topicName, false); + assertEquals(internalStats.numberOfEntries, 0); + }); + + // Make ml.getFirstPosition() return new ledger first position + producer.newMessage().key("K").value("V").send(); + expected.add("V"); + + @Cleanup + ConsumerImpl consumer = (ConsumerImpl) client.newConsumer(Schema.STRING) + .topic(topicName).readCompacted(true).receiverQueueSize(1).subscriptionName(subName) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .isAckReceiptEnabled(true) + .subscribe(); + + List results = new ArrayList<>(); + while (true) { + Message message = consumer.receive(3, TimeUnit.SECONDS); + if (message == null) { + break; + } + + results.add(message.getValue()); + consumer.acknowledge(message); + } + + Assert.assertEquals(results, expected); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java index 4e442ac051326..71700ef83a443 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java @@ -36,6 +36,8 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + import lombok.Cleanup; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.LedgerEntry; @@ -46,10 +48,15 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.admin.LongRunningProcessStatus; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.RawMessage; +import org.apache.pulsar.client.api.Reader; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.RawMessageImpl; @@ -177,6 +184,55 @@ public void testCompaction() throws Exception { compactAndVerify(topic, expected, true); } + @Test + public void testAllCompactedOut() throws Exception { + String topicName = "persistent://my-property/use/my-ns/testAllCompactedOut"; + // set retain null key to true + boolean oldRetainNullKey = pulsar.getConfig().isTopicCompactionRetainNullKey(); + pulsar.getConfig().setTopicCompactionRetainNullKey(true); + this.restartBroker(); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(true).topic(topicName).batchingMaxMessages(3).create(); + + producer.newMessage().key("K1").value("V1").sendAsync(); + producer.newMessage().key("K2").value("V2").sendAsync(); + producer.newMessage().key("K2").value(null).sendAsync(); + producer.flush(); + + admin.topics().triggerCompaction(topicName); + + Awaitility.await().untilAsserted(() -> { + Assert.assertEquals(admin.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS); + }); + + producer.newMessage().key("K1").value(null).sendAsync(); + producer.flush(); + + admin.topics().triggerCompaction(topicName); + + Awaitility.await().untilAsserted(() -> { + Assert.assertEquals(admin.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS); + }); + + @Cleanup + Reader reader = pulsarClient.newReader(Schema.STRING) + .subscriptionName("reader-test") + .topic(topicName) + .readCompacted(true) + .startMessageId(MessageId.earliest) + .create(); + while (reader.hasMessageAvailable()) { + Message message = reader.readNext(3, TimeUnit.SECONDS); + Assert.assertNotNull(message); + } + // set retain null key back to avoid affecting other tests + pulsar.getConfig().setTopicCompactionRetainNullKey(oldRetainNullKey); + } + @Test public void testCompactAddCompact() throws Exception { String topic = "persistent://my-property/use/my-ns/my-topic1"; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/GetLastMessageIdCompactedTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/GetLastMessageIdCompactedTest.java index 317b1a227e585..6c2d848bb7c2d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/GetLastMessageIdCompactedTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/GetLastMessageIdCompactedTest.java @@ -20,6 +20,8 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotEquals; + import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -32,6 +34,7 @@ import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; @@ -415,4 +418,28 @@ public void testGetLastMessageIdAfterCompactionAllNullMsg(boolean enabledBatch) producer.close(); admin.topics().delete(topicName, false); } + + @Test(dataProvider = "enabledBatch") + public void testReaderStuckWithCompaction(boolean enabledBatch) throws Exception { + String topicName = "persistent://public/default/" + BrokerTestUtil.newUniqueName("tp"); + String subName = "sub"; + Producer producer = createProducer(enabledBatch, topicName); + producer.newMessage().key("k0").value("v0").sendAsync(); + producer.newMessage().key("k0").value("v1").sendAsync(); + producer.flush(); + + triggerCompactionAndWait(topicName); + triggerLedgerSwitch(topicName); + clearAllTheLedgersOutdated(topicName); + + var reader = pulsarClient.newReader(Schema.STRING) + .topic(topicName) + .subscriptionName(subName) + .startMessageId(MessageId.earliest) + .create(); + while (reader.hasMessageAvailable()) { + Message message = reader.readNext(5, TimeUnit.SECONDS); + assertNotEquals(message, null); + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionE2ESecurityTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionE2ESecurityTest.java index 107aedd076691..cbf2f28b0b50b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionE2ESecurityTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionE2ESecurityTest.java @@ -34,6 +34,7 @@ import java.net.URL; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Properties; @@ -267,6 +268,11 @@ private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration conf workerConfig.setAuthorizationEnabled(config.isAuthorizationEnabled()); workerConfig.setAuthorizationProvider(config.getAuthorizationProvider()); + List urlPatterns = + List.of(getPulsarApiExamplesJar().getParentFile().toURI() + ".*", "http://127\\.0\\.0\\.1:.*"); + workerConfig.setAdditionalEnabledConnectorUrlPatterns(urlPatterns); + workerConfig.setAdditionalEnabledFunctionsUrlPatterns(urlPatterns); + PulsarWorkerService workerService = new PulsarWorkerService(); return workerService; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java index 9d7493733fe89..569c2d36ff3a7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java @@ -269,6 +269,10 @@ private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration conf workerConfig.setAuthenticationEnabled(true); workerConfig.setAuthorizationEnabled(true); + List urlPatterns = List.of(getPulsarApiExamplesJar().getParentFile().toURI() + ".*"); + workerConfig.setAdditionalEnabledConnectorUrlPatterns(urlPatterns); + workerConfig.setAdditionalEnabledFunctionsUrlPatterns(urlPatterns); + PulsarWorkerService workerService = new PulsarWorkerService(); return workerService; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java index 9882b15450e40..3508cf0bfc7e6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java @@ -20,8 +20,8 @@ import static org.apache.pulsar.common.util.PortManager.nextLockedFreePort; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; @@ -32,6 +32,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -153,6 +154,12 @@ void setup() throws Exception { workerConfig.setUseTls(true); workerConfig.setTlsEnableHostnameVerification(true); workerConfig.setTlsAllowInsecureConnection(false); + File packagePath = new File( + PulsarSink.class.getProtectionDomain().getCodeSource().getLocation().getPath()).getParentFile(); + List urlPatterns = + List.of(packagePath.toURI() + ".*"); + workerConfig.setAdditionalEnabledConnectorUrlPatterns(urlPatterns); + workerConfig.setAdditionalEnabledFunctionsUrlPatterns(urlPatterns); fnWorkerServices[i] = WorkerServiceLoader.load(workerConfig); configurations[i] = config; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java index f968315a7124c..3c0dd0822b7dc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java @@ -35,6 +35,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -306,6 +307,11 @@ private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration conf workerConfig.setAuthenticationEnabled(true); workerConfig.setAuthorizationEnabled(true); + List urlPatterns = + List.of(getPulsarApiExamplesJar().getParentFile().toURI() + ".*", "http://127\\.0\\.0\\.1:.*"); + workerConfig.setAdditionalEnabledConnectorUrlPatterns(urlPatterns); + workerConfig.setAdditionalEnabledFunctionsUrlPatterns(urlPatterns); + PulsarWorkerService workerService = new PulsarWorkerService(); return workerService; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/security/tls/MockedPulsarStandalone.java b/pulsar-broker/src/test/java/org/apache/pulsar/security/MockedPulsarStandalone.java similarity index 76% rename from pulsar-broker/src/test/java/org/apache/pulsar/security/tls/MockedPulsarStandalone.java rename to pulsar-broker/src/test/java/org/apache/pulsar/security/MockedPulsarStandalone.java index 1a7e806f0e698..421727c0ed7f4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/security/tls/MockedPulsarStandalone.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/security/MockedPulsarStandalone.java @@ -16,25 +16,36 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.security.tls; +package org.apache.pulsar.security; import static org.apache.pulsar.utils.ResourceUtils.getAbsolutePath; +import io.jsonwebtoken.Jwts; +import io.jsonwebtoken.SignatureAlgorithm; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import javax.crypto.SecretKey; import lombok.Getter; import lombok.SneakyThrows; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationProviderTls; +import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; +import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; +import org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.impl.auth.AuthenticationKeyStoreTls; import org.apache.pulsar.client.impl.auth.AuthenticationTls; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.common.util.ObjectMapperFactory; + public abstract class MockedPulsarStandalone implements AutoCloseable { @@ -60,6 +71,50 @@ public abstract class MockedPulsarStandalone implements AutoCloseable { serviceConfiguration.setExposeBundlesMetricsInPrometheus(true); } + + protected static final SecretKey SECRET_KEY = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); + + private static final String BROKER_INTERNAL_CLIENT_SUBJECT = "broker_internal"; + private static final String BROKER_INTERNAL_CLIENT_TOKEN = Jwts.builder() + .claim("sub", BROKER_INTERNAL_CLIENT_SUBJECT).signWith(SECRET_KEY).compact(); + protected static final String SUPER_USER_SUBJECT = "super-user"; + protected static final String SUPER_USER_TOKEN = Jwts.builder() + .claim("sub", SUPER_USER_SUBJECT).signWith(SECRET_KEY).compact(); + protected static final String NOBODY_SUBJECT = "nobody"; + protected static final String NOBODY_TOKEN = Jwts.builder() + .claim("sub", NOBODY_SUBJECT).signWith(SECRET_KEY).compact(); + + + @SneakyThrows + protected void configureTokenAuthentication() { + serviceConfiguration.setAuthenticationEnabled(true); + serviceConfiguration.setAuthenticationProviders(Set.of(AuthenticationProviderToken.class.getName())); + // internal client + serviceConfiguration.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); + final Map brokerClientAuthParams = new HashMap<>(); + brokerClientAuthParams.put("token", BROKER_INTERNAL_CLIENT_TOKEN); + final String brokerClientAuthParamStr = MAPPER.writeValueAsString(brokerClientAuthParams); + serviceConfiguration.setBrokerClientAuthenticationParameters(brokerClientAuthParamStr); + + Properties properties = serviceConfiguration.getProperties(); + if (properties == null) { + properties = new Properties(); + serviceConfiguration.setProperties(properties); + } + properties.put("tokenSecretKey", AuthTokenUtils.encodeKeyBase64(SECRET_KEY)); + + } + + + + protected void configureDefaultAuthorization() { + serviceConfiguration.setAuthorizationEnabled(true); + serviceConfiguration.setAuthorizationProvider(PulsarAuthorizationProvider.class.getName()); + serviceConfiguration.setSuperUserRoles(Set.of(SUPER_USER_SUBJECT, BROKER_INTERNAL_CLIENT_SUBJECT)); + } + + + @SneakyThrows protected void loadECTlsCertificateWithFile() { serviceConfiguration.setTlsEnabled(true); @@ -176,4 +231,7 @@ public void close() throws Exception { protected static final String TLS_EC_KS_TRUSTED_STORE = getAbsolutePath("certificate-authority/ec/jks/ca.truststore.jks"); protected static final String TLS_EC_KS_TRUSTED_STORE_PASS = "rootpw"; + + + private static final ObjectMapper MAPPER = ObjectMapperFactory.getMapper().getObjectMapper(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/security/tls/ec/TlsWithECCertificateFileTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/security/tls/ec/TlsWithECCertificateFileTest.java index 39d9b7326d104..b02b10f5996bf 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/security/tls/ec/TlsWithECCertificateFileTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/security/tls/ec/TlsWithECCertificateFileTest.java @@ -25,7 +25,7 @@ import java.util.UUID; import lombok.Cleanup; import lombok.SneakyThrows; -import org.apache.pulsar.security.tls.MockedPulsarStandalone; +import org.apache.pulsar.security.MockedPulsarStandalone; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/security/tls/ec/TlsWithECKeyStoreTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/security/tls/ec/TlsWithECKeyStoreTest.java index e39ad67e4a9d1..c6ff16d4cc50d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/security/tls/ec/TlsWithECKeyStoreTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/security/tls/ec/TlsWithECKeyStoreTest.java @@ -21,9 +21,13 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; import lombok.Cleanup; import lombok.SneakyThrows; -import org.apache.pulsar.security.tls.MockedPulsarStandalone; +import org.apache.pulsar.security.MockedPulsarStandalone; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -35,10 +39,6 @@ import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; @Test diff --git a/pulsar-cli-utils/pom.xml b/pulsar-cli-utils/pom.xml index 4f9f95090f678..bb591028ba9df 100644 --- a/pulsar-cli-utils/pom.xml +++ b/pulsar-cli-utils/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - pulsar-cli-utils Pulsar CLI Utils Isolated CLI utility module - com.beust jcommander compile - org.apache.commons commons-lang3 - @@ -56,7 +50,6 @@ ${pulsar.client.compiler.release} - org.gaul modernizer-maven-plugin @@ -74,7 +67,6 @@ - pl.project13.maven git-commit-id-plugin @@ -99,7 +91,6 @@ - org.codehaus.mojo templating-maven-plugin @@ -113,7 +104,6 @@ - com.github.spotbugs spotbugs-maven-plugin diff --git a/pulsar-cli-utils/src/main/resources/findbugsExclude.xml b/pulsar-cli-utils/src/main/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-cli-utils/src/main/resources/findbugsExclude.xml +++ b/pulsar-cli-utils/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-client-1x-base/pom.xml b/pulsar-client-1x-base/pom.xml index d3f099d639374..871b9bd4e9189 100644 --- a/pulsar-client-1x-base/pom.xml +++ b/pulsar-client-1x-base/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - pulsar-client-1x-base Pulsar Client 1.x Compatibility Base pom - pulsar-client-2x-shaded pulsar-client-1x - @@ -61,7 +57,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -77,5 +72,4 @@ - diff --git a/pulsar-client-1x-base/pulsar-client-1x/pom.xml b/pulsar-client-1x-base/pulsar-client-1x/pom.xml index c3dc3068847bd..79fb1040489f8 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-1x/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-client-1x-base 3.2.0-SNAPSHOT .. - pulsar-client-1x Pulsar Client 1.x Compatibility API - ${project.groupId} pulsar-client-2x-shaded ${project.version} - com.google.guava guava - org.apache.commons commons-lang3 - - @@ -70,7 +63,6 @@ - com.github.spotbugs spotbugs-maven-plugin @@ -90,5 +82,4 @@ - diff --git a/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/ClientConfiguration.java b/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/ClientConfiguration.java index 3b0efe64cf588..ea2bba166e6c5 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/ClientConfiguration.java +++ b/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/ClientConfiguration.java @@ -368,6 +368,7 @@ public ClientConfiguration setServiceUrl(String serviceUrl) { * @param unit the time unit in which the duration is defined */ public void setConnectionTimeout(int duration, TimeUnit unit) { + checkArgument(duration >= 0); confData.setConnectionTimeoutMs((int) unit.toMillis(duration)); } diff --git a/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/ConsumerConfiguration.java b/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/ConsumerConfiguration.java index 81956db56f774..f2101b287043c 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/ConsumerConfiguration.java +++ b/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/ConsumerConfiguration.java @@ -69,6 +69,7 @@ public long getAckTimeoutMillis() { * @return {@link ConsumerConfiguration} */ public ConsumerConfiguration setAckTimeout(long ackTimeout, TimeUnit timeUnit) { + checkArgument(ackTimeout >= 0); long ackTimeoutMillis = timeUnit.toMillis(ackTimeout); checkArgument(ackTimeoutMillis >= minAckTimeoutMillis, "Ack timeout should be should be greater than " + minAckTimeoutMillis + " ms"); diff --git a/pulsar-client-1x-base/pulsar-client-1x/src/main/resources/findbugsExclude.xml b/pulsar-client-1x-base/pulsar-client-1x/src/main/resources/findbugsExclude.xml index 7938e60bf4330..d191e191d52ca 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/src/main/resources/findbugsExclude.xml +++ b/pulsar-client-1x-base/pulsar-client-1x/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-client-1x-base 3.2.0-SNAPSHOT .. - pulsar-client-2x-shaded Pulsar Client 2.x Shaded API - ${project.groupId} @@ -39,10 +36,9 @@ ${project.version} - - + org.apache.maven.plugins maven-shade-plugin @@ -56,16 +52,15 @@ true true false - - org.apache.pulsar:pulsar-client - org.apache.pulsar:pulsar-client-api + io.streamnative:pulsar-client + io.streamnative:pulsar-client-api - org.apache.pulsar:pulsar-client + io.streamnative:pulsar-client ** @@ -93,6 +88,30 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/pulsar-client-admin-api/pom.xml b/pulsar-client-admin-api/pom.xml index 5e4b86fa283b5..0cd67bea4998a 100644 --- a/pulsar-client-admin-api/pom.xml +++ b/pulsar-client-admin-api/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - - org.apache.pulsar - pulsar - 3.2.0-SNAPSHOT - .. - - - pulsar-client-admin-api - Pulsar Client Admin :: API - - - - ${project.groupId} - pulsar-client-api - ${project.version} - - - - org.slf4j - slf4j-api - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - ${pulsar.client.compiler.release} - - - - org.gaul - modernizer-maven-plugin - - true - 8 - - - - modernizer - verify - - modernizer - - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - - + + 4.0.0 + + io.streamnative + pulsar + 3.2.0-SNAPSHOT + .. + + pulsar-client-admin-api + Pulsar Client Admin :: API + + + ${project.groupId} + pulsar-client-api + ${project.version} + + + org.slf4j + slf4j-api + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${pulsar.client.compiler.release} + + + + org.gaul + modernizer-maven-plugin + + true + 8 + + + + modernizer + verify + + modernizer + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Brokers.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Brokers.java index 29c280f8ba536..dc0b7c9885a9a 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Brokers.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Brokers.java @@ -35,7 +35,7 @@ public interface Brokers { /** * Get the list of active brokers in the local cluster. *

- * Get the list of active brokers (web service addresses) in the local cluster. + * Get the list of active brokers (broker ids) in the local cluster. *

* Response Example: * @@ -44,7 +44,7 @@ public interface Brokers { * * * "prod1-broker3.messaging.use.example.com:8080"] * * - * @return a list of (host:port) + * @return a list of broker ids * @throws NotAuthorizedException * You don't have admin permission to get the list of active brokers in the cluster * @throws PulsarAdminException @@ -55,7 +55,7 @@ public interface Brokers { /** * Get the list of active brokers in the local cluster asynchronously. *

- * Get the list of active brokers (web service addresses) in the local cluster. + * Get the list of active brokers (broker ids) in the local cluster. *

* Response Example: * @@ -64,13 +64,13 @@ public interface Brokers { * "prod1-broker3.messaging.use.example.com:8080"] * * - * @return a list of (host:port) + * @return a list of broker ids */ CompletableFuture> getActiveBrokersAsync(); /** * Get the list of active brokers in the cluster. *

- * Get the list of active brokers (web service addresses) in the cluster. + * Get the list of active brokers (broker ids) in the cluster. *

* Response Example: * @@ -81,7 +81,7 @@ public interface Brokers { * * @param cluster * Cluster name - * @return a list of (host:port) + * @return a list of broker ids * @throws NotAuthorizedException * You don't have admin permission to get the list of active brokers in the cluster * @throws NotFoundException @@ -94,7 +94,7 @@ public interface Brokers { /** * Get the list of active brokers in the cluster asynchronously. *

- * Get the list of active brokers (web service addresses) in the cluster. + * Get the list of active brokers (broker ids) in the cluster. *

* Response Example: * @@ -105,7 +105,7 @@ public interface Brokers { * * @param cluster * Cluster name - * @return a list of (host:port) + * @return a list of broker ids */ CompletableFuture> getActiveBrokersAsync(String cluster); @@ -156,11 +156,11 @@ public interface Brokers { * * * @param cluster - * @param brokerUrl + * @param brokerId * @return * @throws PulsarAdminException */ - Map getOwnedNamespaces(String cluster, String brokerUrl) + Map getOwnedNamespaces(String cluster, String brokerId) throws PulsarAdminException; /** @@ -176,10 +176,10 @@ Map getOwnedNamespaces(String cluster, String * * * @param cluster - * @param brokerUrl + * @param brokerId * @return */ - CompletableFuture> getOwnedNamespacesAsync(String cluster, String brokerUrl); + CompletableFuture> getOwnedNamespacesAsync(String cluster, String brokerId); /** * Update a dynamic configuration value into ZooKeeper. diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/BrokerInfo.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/BrokerInfo.java index 8955fe7a0ac78..19e9ff2d15a2b 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/BrokerInfo.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/BrokerInfo.java @@ -25,9 +25,11 @@ */ public interface BrokerInfo { String getServiceUrl(); + String getBrokerId(); interface Builder { Builder serviceUrl(String serviceUrl); + Builder brokerId(String brokerId); BrokerInfo build(); } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/BrokerInfoImpl.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/BrokerInfoImpl.java index e4d0a68b50ad0..d77f693c7cd70 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/BrokerInfoImpl.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/BrokerInfoImpl.java @@ -31,6 +31,7 @@ @NoArgsConstructor public final class BrokerInfoImpl implements BrokerInfo { private String serviceUrl; + private String brokerId; public static BrokerInfoImplBuilder builder() { return new BrokerInfoImplBuilder(); @@ -38,14 +39,20 @@ public static BrokerInfoImplBuilder builder() { public static class BrokerInfoImplBuilder implements BrokerInfo.Builder { private String serviceUrl; + private String brokerId; public BrokerInfoImplBuilder serviceUrl(String serviceUrl) { this.serviceUrl = serviceUrl; return this; } + public BrokerInfoImplBuilder brokerId(String brokerId) { + this.brokerId = brokerId; + return this; + } + public BrokerInfoImpl build() { - return new BrokerInfoImpl(serviceUrl); + return new BrokerInfoImpl(serviceUrl, brokerId); } } } diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index 49f4e7772780b..5d1be42e57aa2 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - pulsar-client-admin Pulsar Client Admin - ${project.groupId} @@ -74,7 +70,6 @@ - maven-antrun-plugin @@ -87,15 +82,12 @@ - + - org.apache.maven.plugins maven-shade-plugin @@ -108,11 +100,10 @@ true true - - org.apache.pulsar:pulsar-client-original - org.apache.pulsar:pulsar-client-admin-original + io.streamnative:pulsar-client-original + io.streamnative:pulsar-client-admin-original org.apache.commons:commons-lang3 commons-codec:commons-codec commons-collections:commons-collections @@ -126,7 +117,7 @@ com.fasterxml.jackson.*:* io.netty:* io.netty.incubator:* - org.apache.pulsar:pulsar-common + io.streamnative:pulsar-common org.apache.bookkeeper:* com.yahoo.datasketches:sketches-core org.glassfish.jersey*:* @@ -148,7 +139,7 @@ org.yaml:snakeyaml io.swagger:* - org.apache.pulsar:pulsar-client-messagecrypto-bc + io.streamnative:pulsar-client-messagecrypto-bc com.fasterxml.jackson.core:jackson-annotations @@ -156,7 +147,7 @@ - org.apache.pulsar:pulsar-client-original + io.streamnative:pulsar-client-original ** @@ -166,7 +157,7 @@ - org.apache.pulsar:pulsar-client-admin-original + io.streamnative:pulsar-client-admin-original ** @@ -177,7 +168,7 @@ - + org.asynchttpclient org.apache.pulsar.shade.org.asynchttpclient @@ -259,51 +250,75 @@ org.reactivestreams org.apache.pulsar.shade.org.reactivestreams - - io.grpc - org.apache.pulsar.shade.io.grpc - - - okio - org.apache.pulsar.shade.okio - - - com.squareup - org.apache.pulsar.shade.com.squareup - - - io.opencensus - org.apache.pulsar.shade.io.opencensus - - - org.eclipse.jetty - org.apache.pulsar.shade.org.eclipse.jetty - - - org.objenesis - org.apache.pulsar.shade.org.objenesis - - - org.yaml - org.apache.pulsar.shade.org.yaml - - - io.swagger - org.apache.pulsar.shade.io.swagger - - - org.apache.bookkeeper - org.apache.pulsar.shade.org.apache.bookkeeper - + + io.grpc + org.apache.pulsar.shade.io.grpc + + + okio + org.apache.pulsar.shade.okio + + + com.squareup + org.apache.pulsar.shade.com.squareup + + + io.opencensus + org.apache.pulsar.shade.io.opencensus + + + org.eclipse.jetty + org.apache.pulsar.shade.org.eclipse.jetty + + + org.objenesis + org.apache.pulsar.shade.org.objenesis + + + org.yaml + org.apache.pulsar.shade.org.yaml + + + io.swagger + org.apache.pulsar.shade.io.swagger + + + org.apache.bookkeeper + org.apache.pulsar.shade.org.apache.bookkeeper + - - + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/pulsar-client-admin/pom.xml b/pulsar-client-admin/pom.xml index e349f1a8b9f6d..c1cd8fe56d2d2 100644 --- a/pulsar-client-admin/pom.xml +++ b/pulsar-client-admin/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - pulsar-client-admin-original Pulsar Client Admin Original - ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} pulsar-client-admin-api ${project.version} - org.glassfish.jersey.core jersey-client - org.glassfish.jersey.media jersey-media-json-jackson - jakarta.activation jakarta.activation-api - org.glassfish.jersey.media jersey-media-multipart - com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - org.glassfish.jersey.inject jersey-hk2 - javax.xml.bind jaxb-api @@ -91,30 +79,25 @@ javax.activation runtime - com.google.guava guava - com.google.code.gson gson - ${project.groupId} pulsar-package-core ${project.version} - org.hamcrest hamcrest test - @@ -141,7 +124,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BrokersImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BrokersImpl.java index 0e6296724b3da..7b4ebb1778d8e 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BrokersImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BrokersImpl.java @@ -75,15 +75,15 @@ public CompletableFuture getLeaderBrokerAsync() { } @Override - public Map getOwnedNamespaces(String cluster, String brokerUrl) + public Map getOwnedNamespaces(String cluster, String brokerId) throws PulsarAdminException { - return sync(() -> getOwnedNamespacesAsync(cluster, brokerUrl)); + return sync(() -> getOwnedNamespacesAsync(cluster, brokerId)); } @Override public CompletableFuture> getOwnedNamespacesAsync( - String cluster, String brokerUrl) { - WebTarget path = adminBrokers.path(cluster).path(brokerUrl).path("ownedNamespaces"); + String cluster, String brokerId) { + WebTarget path = adminBrokers.path(cluster).path(brokerId).path("ownedNamespaces"); return asyncGetRequest(path, new FutureCallback>(){}); } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java index 009fa67fbaa29..a9d913c016490 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.admin.internal; +import static com.google.common.base.Preconditions.checkArgument; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -200,18 +201,21 @@ public PulsarAdminBuilder tlsProtocols(Set tlsProtocols) { @Override public PulsarAdminBuilder connectionTimeout(int connectionTimeout, TimeUnit connectionTimeoutUnit) { + checkArgument(connectionTimeout >= 0); this.conf.setConnectionTimeoutMs((int) connectionTimeoutUnit.toMillis(connectionTimeout)); return this; } @Override public PulsarAdminBuilder readTimeout(int readTimeout, TimeUnit readTimeoutUnit) { + checkArgument(readTimeout >= 0); this.conf.setReadTimeoutMs((int) readTimeoutUnit.toMillis(readTimeout)); return this; } @Override public PulsarAdminBuilder requestTimeout(int requestTimeout, TimeUnit requestTimeoutUnit) { + checkArgument(requestTimeout >= 0); this.conf.setRequestTimeoutMs((int) requestTimeoutUnit.toMillis(requestTimeout)); return this; } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java index 9d09d96073d9e..39bbb134271f1 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java @@ -987,21 +987,7 @@ public CompletableFuture truncateAsync(String topic) { @Override public CompletableFuture> getMessageByIdAsync(String topic, long ledgerId, long entryId) { - CompletableFuture> future = new CompletableFuture<>(); - getRemoteMessageById(topic, ledgerId, entryId).handle((r, ex) -> { - if (ex != null) { - if (ex instanceof NotFoundException) { - log.warn("Exception '{}' occurred while trying to get message.", ex.getMessage()); - future.complete(r); - } else { - future.completeExceptionally(ex); - } - return null; - } - future.complete(r); - return null; - }); - return future; + return getRemoteMessageById(topic, ledgerId, entryId); } private CompletableFuture> getRemoteMessageById(String topic, long ledgerId, long entryId) { diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TransactionsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TransactionsImpl.java index 2d1dd408ef6c9..046ef3eb370e8 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TransactionsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TransactionsImpl.java @@ -168,6 +168,7 @@ public TransactionPendingAckStats getPendingAckStats(String topic, String subNam @Override public CompletableFuture> getSlowTransactionsByCoordinatorIdAsync( Integer coordinatorId, long timeout, TimeUnit timeUnit) { + checkArgument(timeout >= 0); WebTarget path = adminV3Transactions.path("slowTransactions"); path = path.path(timeUnit.toMillis(timeout) + ""); if (coordinatorId != null) { diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index e127ed453d5f4..95666e24d6f15 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - pulsar-client-all Pulsar Client All - ${project.groupId} @@ -70,7 +67,6 @@ test - @@ -108,7 +104,6 @@ - maven-antrun-plugin @@ -121,15 +116,12 @@ - + - org.apache.maven.plugins @@ -146,11 +138,10 @@ true true false - - org.apache.pulsar:pulsar-client-original - org.apache.pulsar:pulsar-client-admin-original + io.streamnative:pulsar-client-original + io.streamnative:pulsar-client-admin-original org.apache.commons:commons-lang3 commons-codec:commons-codec commons-collections:commons-collections @@ -176,8 +167,7 @@ commons-*:* io.swagger:* io.airlift:* - - org.apache.pulsar:pulsar-common + io.streamnative:pulsar-common org.apache.bookkeeper:* com.yahoo.datasketches:sketches-core org.glassfish.jersey*:* @@ -205,7 +195,7 @@ org.apache.commons:commons-compress org.tukaani:xz - org.apache.pulsar:pulsar-client-messagecrypto-bc + io.streamnative:pulsar-client-messagecrypto-bc com.fasterxml.jackson.core:jackson-annotations @@ -213,7 +203,7 @@ - org.apache.pulsar:pulsar-client-original + io.streamnative:pulsar-client-original ** @@ -315,24 +305,24 @@ org.apache.pulsar.shade.org.glassfish - io.grpc - org.apache.pulsar.shade.io.grpc + io.grpc + org.apache.pulsar.shade.io.grpc - okio - org.apache.pulsar.shade.okio + okio + org.apache.pulsar.shade.okio - com.squareup - org.apache.pulsar.shade.com.squareup + com.squareup + org.apache.pulsar.shade.com.squareup - io.opencensus - org.apache.pulsar.shade.io.opencensus + io.opencensus + org.apache.pulsar.shade.io.opencensus - org.eclipse.jetty - org.apache.pulsar.shade.org.eclipse.jetty + org.eclipse.jetty + org.apache.pulsar.shade.org.eclipse.jetty org.objenesis @@ -389,14 +379,13 @@ - - + + - - - 4.0.0 - - - org.apache.pulsar - pulsar - 3.2.0-SNAPSHOT - .. - - - pulsar-client-api - Pulsar Client :: API - - - - - - com.google.protobuf - protobuf-java - provided - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - ${pulsar.client.compiler.release} - - - - org.gaul - modernizer-maven-plugin - - true - 8 - - - - modernizer - verify - - modernizer - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - - + + com.google.protobuf + protobuf-java + provided + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${pulsar.client.compiler.release} + + + + org.gaul + modernizer-maven-plugin + + true + 8 + + + + modernizer + verify + + modernizer + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java index 01f205a3afde5..ddb414a72d173 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java @@ -126,7 +126,7 @@ public interface ConsumerBuilder extends Cloneable { ConsumerBuilder topics(List topicNames); /** - * Specify a pattern for topics that this consumer subscribes to. + * Specify a pattern for topics(not contains the partition suffix) that this consumer subscribes to. * *

The pattern is applied to subscribe to all topics, within a single namespace, that match the * pattern. @@ -134,13 +134,13 @@ public interface ConsumerBuilder extends Cloneable { *

The consumer automatically subscribes to topics created after itself. * * @param topicsPattern - * a regular expression to select a list of topics to subscribe to + * a regular expression to select a list of topics(not contains the partition suffix) to subscribe to * @return the consumer builder instance */ ConsumerBuilder topicsPattern(Pattern topicsPattern); /** - * Specify a pattern for topics that this consumer subscribes to. + * Specify a pattern for topics(not contains the partition suffix) that this consumer subscribes to. * *

It accepts a regular expression that is compiled into a pattern internally. E.g., * "persistent://public/default/pattern-topic-.*" @@ -151,7 +151,7 @@ public interface ConsumerBuilder extends Cloneable { *

The consumer automatically subscribes to topics created after itself. * * @param topicsPattern - * given regular expression for topics pattern + * given regular expression for topics(not contains the partition suffix) pattern * @return the consumer builder instance */ ConsumerBuilder topicsPattern(String topicsPattern); diff --git a/pulsar-client-api/src/main/resources/findbugsExclude.xml b/pulsar-client-api/src/main/resources/findbugsExclude.xml index 9d73ac29a7bdb..353f01a7bb285 100644 --- a/pulsar-client-api/src/main/resources/findbugsExclude.xml +++ b/pulsar-client-api/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml index e0a1b29b479e6..1e60cd1a281dc 100644 --- a/pulsar-client-auth-athenz/pom.xml +++ b/pulsar-client-auth-athenz/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - pulsar-client-auth-athenz jar Athenz authentication plugin for java client - - ${project.groupId} pulsar-client-original ${project.parent.version} true - com.yahoo.athenz athenz-zts-java-client-core - com.yahoo.athenz athenz-cert-refresher - org.bouncycastle bcpkix-jdk18on - com.google.guava guava - org.apache.commons commons-lang3 - @@ -77,7 +67,6 @@ ${pulsar.client.compiler.release} - org.apache.maven.plugins maven-enforcer-plugin @@ -108,7 +97,6 @@ - org.gaul modernizer-maven-plugin @@ -126,7 +114,6 @@ - com.github.spotbugs spotbugs-maven-plugin @@ -144,7 +131,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-client-auth-athenz/src/test/resources/findbugsExclude.xml b/pulsar-client-auth-athenz/src/test/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-client-auth-athenz/src/test/resources/findbugsExclude.xml +++ b/pulsar-client-auth-athenz/src/test/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - pulsar-client-auth-sasl jar SASL authentication plugin for java client - - ${project.groupId} pulsar-client-original ${project.parent.version} true - com.google.guava guava - org.apache.commons commons-lang3 - org.projectlombok lombok - javax.ws.rs javax.ws.rs-api - org.glassfish.jersey.core jersey-client - - @@ -78,7 +67,6 @@ ${pulsar.client.compiler.release} - org.apache.maven.plugins maven-enforcer-plugin @@ -109,7 +97,6 @@ - org.gaul modernizer-maven-plugin @@ -127,7 +114,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-client-messagecrypto-bc/pom.xml b/pulsar-client-messagecrypto-bc/pom.xml index b1038a897957d..7d5bc1a3bffcd 100644 --- a/pulsar-client-messagecrypto-bc/pom.xml +++ b/pulsar-client-messagecrypto-bc/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - pulsar-client-messagecrypto-bc jar Message crypto for End to End encryption with BouncyCastleProvider - ${project.groupId} @@ -40,7 +37,6 @@ ${project.parent.version} provided - ${project.groupId} bouncy-castle-bc @@ -48,9 +44,7 @@ pkg true - - diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml index c4ab92634e0b8..d02eed2ed4f0d 100644 --- a/pulsar-client-shaded/pom.xml +++ b/pulsar-client-shaded/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - pulsar-client Pulsar Client Java - ${project.groupId} @@ -49,11 +46,8 @@ ${project.version} - - - org.apache.maven.plugins maven-dependency-plugin @@ -89,7 +83,6 @@ - maven-antrun-plugin @@ -102,15 +95,12 @@ - + - org.apache.maven.plugins @@ -125,10 +115,9 @@ true true false - - org.apache.pulsar:pulsar-client-original + io.streamnative:pulsar-client-original org.apache.bookkeeper:* org.apache.commons:commons-lang3 commons-codec:commons-codec @@ -154,19 +143,17 @@ commons-*:* io.swagger:* io.airlift:* - - org.apache.pulsar:pulsar-common + io.streamnative:pulsar-common com.yahoo.datasketches:sketches-core org.objenesis:* org.yaml:snakeyaml - org.apache.avro:* com.thoughtworks.paranamer:paranamer org.apache.commons:commons-compress org.tukaani:xz - org.apache.pulsar:pulsar-client-messagecrypto-bc + io.streamnative:pulsar-client-messagecrypto-bc com.fasterxml.jackson.core:jackson-annotations @@ -174,7 +161,7 @@ - org.apache.pulsar:pulsar-client-original + io.streamnative:pulsar-client-original ** @@ -302,14 +289,13 @@ - - + + - com.github.spotbugs spotbugs-maven-plugin @@ -323,7 +309,6 @@ - - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - pulsar-client-tools-api Pulsar Client Tools API Pulsar Client Tools API - ${project.groupId} @@ -66,7 +64,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -82,5 +79,4 @@ - diff --git a/pulsar-client-tools-customcommand-example/pom.xml b/pulsar-client-tools-customcommand-example/pom.xml index f27ccdb337b34..5bfa8067a4c39 100644 --- a/pulsar-client-tools-customcommand-example/pom.xml +++ b/pulsar-client-tools-customcommand-example/pom.xml @@ -1,3 +1,4 @@ + - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. @@ -31,7 +32,7 @@ Pulsar CLI Custom command example - org.apache.pulsar + io.streamnative pulsar-client-tools-api ${project.version} provided diff --git a/pulsar-client-tools-test/pom.xml b/pulsar-client-tools-test/pom.xml index 9814ceb9a0a52..a6f0e217b28fa 100644 --- a/pulsar-client-tools-test/pom.xml +++ b/pulsar-client-tools-test/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - pulsar-client-tools-test Pulsar Client Tools Test Pulsar Client Tools Test - ${project.groupId} @@ -89,7 +87,6 @@ - org.apache.maven.plugins maven-deploy-plugin @@ -126,8 +123,8 @@ copy filters - - + + diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java index 07efa5165e5fd..fdcf6bdac35f5 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java @@ -468,6 +468,11 @@ public void namespaces() throws Exception { namespaces.run(split("remove-replicator-dispatch-rate myprop/clust/ns1")); verify(mockNamespaces).removeReplicatorDispatchRate("myprop/clust/ns1"); + + assertFalse(namespaces.run(split("unload myprop/clust/ns1 -d broker"))); + verify(mockNamespaces, times(0)).unload("myprop/clust/ns1"); + + namespaces = new CmdNamespaces(() -> admin); namespaces.run(split("unload myprop/clust/ns1")); verify(mockNamespaces).unload("myprop/clust/ns1"); @@ -484,6 +489,10 @@ public void namespaces() throws Exception { namespaces.run(split("unload myprop/clust/ns1 -b 0x80000000_0xffffffff")); verify(mockNamespaces).unloadNamespaceBundle("myprop/clust/ns1", "0x80000000_0xffffffff", null); + namespaces = new CmdNamespaces(() -> admin); + namespaces.run(split("unload myprop/clust/ns1 -b 0x80000000_0xffffffff -d broker")); + verify(mockNamespaces).unloadNamespaceBundle("myprop/clust/ns1", "0x80000000_0xffffffff", "broker"); + namespaces.run(split("split-bundle myprop/clust/ns1 -b 0x00000000_0xffffffff")); verify(mockNamespaces).splitNamespaceBundle("myprop/clust/ns1", "0x00000000_0xffffffff", false, null); diff --git a/pulsar-client-tools-test/src/test/resources/findbugsExclude.xml b/pulsar-client-tools-test/src/test/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-client-tools-test/src/test/resources/findbugsExclude.xml +++ b/pulsar-client-tools-test/src/test/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - pulsar-client-tools Pulsar Client Tools Pulsar Client Tools - com.beust @@ -72,7 +70,7 @@ pulsar-client-messagecrypto-bc ${project.version} - + ${project.groupId} pulsar-cli-utils ${project.version} @@ -119,7 +117,6 @@ swagger-core provided - ${project.groupId} @@ -127,9 +124,8 @@ ${project.version} test - - org.apache.pulsar + io.streamnative pulsar-io-batch-discovery-triggerers ${project.version} test @@ -147,9 +143,7 @@ org.apache.commons commons-text - - @@ -169,7 +163,6 @@ - @@ -189,7 +182,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -205,5 +197,4 @@ - diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index c7658cbf77055..6dfeaa77298d5 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -752,8 +752,8 @@ private class SetRetention extends CliCommand { + "For example, 4096, 10M, 16G, 3T. The size unit suffix character can be k/K, m/M, g/G, or t/T. " + "If the size unit suffix is not specified, the default unit is bytes. " + "0 or less than 1MB means no retention and -1 means infinite size retention", required = true, - converter = ByteUnitIntegerConverter.class) - private Integer sizeLimit; + converter = ByteUnitToLongConverter.class) + private Long sizeLimit; @Override void run() throws PulsarAdminException { @@ -761,8 +761,8 @@ void run() throws PulsarAdminException { final int retentionTimeInMin = retentionTimeInSec != -1 ? (int) TimeUnit.SECONDS.toMinutes(retentionTimeInSec) : retentionTimeInSec.intValue(); - final int retentionSizeInMB = sizeLimit != -1 - ? (int) (sizeLimit / (1024 * 1024)) + final long retentionSizeInMB = sizeLimit != -1 + ? (sizeLimit / (1024 * 1024)) : sizeLimit; getAdmin().namespaces() .setRetention(namespace, new RetentionPolicies(retentionTimeInMin, retentionSizeInMB)); @@ -866,13 +866,19 @@ private class Unload extends CliCommand { private String bundle; @Parameter(names = { "--destinationBroker", "-d" }, - description = "Target brokerWebServiceAddress to which the bundle has to be allocated to") + description = "Target brokerWebServiceAddress to which the bundle has to be allocated to. " + + "--destinationBroker cannot be set when --bundle is not specified.") private String destinationBroker; @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); + + if (bundle == null) { + if (StringUtils.isNotBlank(destinationBroker)) { + throw new ParameterException("--destinationBroker cannot be set when --bundle is not specified."); + } getAdmin().namespaces().unload(namespace); } else { getAdmin().namespaces().unloadNamespaceBundle(namespace, bundle, destinationBroker); @@ -1929,7 +1935,8 @@ private class GetOffloadThreshold extends CliCommand { @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); - print(getAdmin().namespaces().getOffloadThreshold(namespace)); + print("offloadThresholdInBytes: " + getAdmin().namespaces().getOffloadThreshold(namespace)); + print("offloadThresholdInSeconds: " + getAdmin().namespaces().getOffloadThresholdInSeconds(namespace)); } } @@ -1948,10 +1955,17 @@ private class SetOffloadThreshold extends CliCommand { converter = ByteUnitToLongConverter.class) private Long threshold = -1L; + @Parameter(names = {"--time", "-t"}, + description = "Maximum number of seconds stored on the pulsar cluster for a topic" + + " before the broker will start offloading to longterm storage (eg: 10m, 5h, 3d, 2w).", + converter = TimeUnitToSecondsConverter.class) + private Long thresholdInSeconds = -1L; + @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); getAdmin().namespaces().setOffloadThreshold(namespace, threshold); + getAdmin().namespaces().setOffloadThresholdInSeconds(namespace, thresholdInSeconds); } } diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdNamespaces.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdNamespaces.java new file mode 100644 index 0000000000000..f9ce84411c6c0 --- /dev/null +++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdNamespaces.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli; + +import org.apache.pulsar.client.admin.Namespaces; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.common.policies.data.RetentionPolicies; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.Test; +import java.io.IOException; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class TestCmdNamespaces { + + @AfterMethod(alwaysRun = true) + public void cleanup() throws IOException { + //NOTHING FOR NOW + } + + + @Test + public void testSetRetentionCmd() throws Exception { + Namespaces namespaces = mock(Namespaces.class); + + PulsarAdmin admin = mock(PulsarAdmin.class); + when(admin.namespaces()).thenReturn(namespaces); + + CmdNamespaces cmd = new CmdNamespaces(() -> admin); + + cmd.run("set-retention public/default -s 2T -t 2h".split("\\s+")); + verify(namespaces, times(1)).setRetention("public/default", new RetentionPolicies(120, 2 * 1024 * 1024)); + } +} diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index 4307b4bbac737..5dcb079318bb9 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - pulsar-client-original Pulsar Client Java - ${project.groupId} pulsar-client-api ${project.parent.version} - ${project.groupId} pulsar-common ${project.parent.version} - ${project.groupId} bouncy-castle-bc ${project.parent.version} pkg - ${project.groupId} @@ -59,7 +53,6 @@ ${project.parent.version} true - io.netty netty-codec-http @@ -72,13 +65,11 @@ io.netty netty-codec-socks - io.swagger swagger-annotations provided - io.netty netty-resolver-dns @@ -93,50 +84,40 @@ netty-resolver-dns-native-macos osx-x86_64 - org.apache.commons commons-lang3 - org.asynchttpclient async-http-client - com.typesafe.netty netty-reactive-streams - org.slf4j slf4j-api - commons-codec commons-codec - com.yahoo.datasketches sketches-core - com.google.code.gson gson - - org.apache.avro avro ${avro.version} - org.apache.avro avro-protobuf @@ -148,36 +129,30 @@ - joda-time joda-time provided - com.google.protobuf protobuf-java provided - com.fasterxml.jackson.module jackson-module-jsonSchema - net.jcip jcip-annotations - com.github.spotbugs spotbugs-annotations provided true - ${project.groupId} @@ -185,22 +160,18 @@ ${project.parent.version} test - org.skyscreamer jsonassert ${skyscreamer.version} test - org.awaitility awaitility test - - @@ -234,7 +205,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -248,7 +218,6 @@ - org.xolstice.maven.plugins protobuf-maven-plugin diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java index e81365d3886cc..8c17d8fcb253c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java @@ -53,6 +53,7 @@ public abstract class AbstractBatchMessageContainer implements BatchMessageConta // allocate a new buffer that can hold the entire batch without needing costly reallocations protected int maxBatchSize = INITIAL_BATCH_BUFFER_SIZE; protected int maxMessagesNum = INITIAL_MESSAGES_NUM; + private volatile long firstAddedTimestamp = 0L; @Override public boolean haveEnoughSpace(MessageImpl msg) { @@ -127,4 +128,19 @@ public boolean hasSameTxn(MessageImpl msg) { return currentTxnidMostBits == msg.getMessageBuilder().getTxnidMostBits() && currentTxnidLeastBits == msg.getMessageBuilder().getTxnidLeastBits(); } + + @Override + public long getFirstAddedTimestamp() { + return firstAddedTimestamp; + } + + protected void tryUpdateTimestamp() { + if (numMessagesInBatch == 1) { + firstAddedTimestamp = System.nanoTime(); + } + } + + protected void clearTimestamp() { + firstAddedTimestamp = 0L; + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java index 68b781e67d29c..a1017e66760a5 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java @@ -329,6 +329,7 @@ public AutoClusterFailoverBuilder switchBackDelay(long switchBackDelay, TimeUnit @Override public AutoClusterFailoverBuilder checkInterval(long interval, TimeUnit timeUnit) { + checkArgument(interval >= 0L, "check interval time must not be negative."); this.checkIntervalMs = timeUnit.toMillis(interval); return this; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerBase.java index 8fb4e9f2ce543..ddbe1bc255779 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerBase.java @@ -82,4 +82,11 @@ public interface BatchMessageContainerBase extends BatchMessageContainer { * @return belong to the same txn or not */ boolean hasSameTxn(MessageImpl msg); + + /** + * Get the timestamp in nanoseconds when the 1st message is added into the batch container. + * + * @return the timestamp in nanoseconds or 0L if the batch container is empty + */ + long getFirstAddedTimestamp(); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java index dfcbc42bcc6b8..bf8c1f9de8201 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java @@ -127,6 +127,7 @@ public boolean add(MessageImpl msg, SendCallback callback) { previousCallback = callback; currentBatchSizeBytes += msg.getDataBuffer().readableBytes(); messages.add(msg); + tryUpdateTimestamp(); if (lowestSequenceId == -1L) { lowestSequenceId = msg.getSequenceId(); @@ -203,6 +204,7 @@ void updateMaxBatchSize(int uncompressedSize) { @Override public void clear() { + clearTimestamp(); messages = new ArrayList<>(maxMessagesNum); firstCallback = null; previousCallback = null; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java index e2ce9e2d0bd70..1592d3cae6cb5 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java @@ -57,11 +57,13 @@ public boolean add(MessageImpl msg, SendCallback callback) { numMessagesInBatch++; currentBatchSizeBytes += msg.getDataBuffer().readableBytes(); } + tryUpdateTimestamp(); return isBatchFull(); } @Override public void clear() { + clearTimestamp(); numMessagesInBatch = 0; currentBatchSizeBytes = 0; batches.clear(); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java index 9a86d81c93fab..48b3b2a6bda62 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java @@ -168,6 +168,7 @@ public ClientBuilder operationTimeout(int operationTimeout, TimeUnit unit) { @Override public ClientBuilder lookupTimeout(int lookupTimeout, TimeUnit unit) { + checkArgument(lookupTimeout >= 0, "lookupTimeout must not be negative"); conf.setLookupTimeoutMs(unit.toMillis(lookupTimeout)); return this; } @@ -333,6 +334,7 @@ public ClientBuilder keepAliveInterval(int keepAliveInterval, TimeUnit unit) { @Override public ClientBuilder connectionTimeout(int duration, TimeUnit unit) { + checkArgument(duration >= 0, "connectionTimeout needs to be >= 0"); conf.setConnectionTimeoutMs((int) unit.toMillis(duration)); return this; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index 4f29c0aa76c94..d45c125c590de 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -64,6 +64,7 @@ import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.collections.BitSetRecyclable; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.common.util.collections.GrowableArrayBlockingQueue; import org.slf4j.Logger; @@ -448,6 +449,7 @@ public void acknowledge(Messages messages) throws PulsarClientException { @Override public void reconsumeLater(Message message, long delayTime, TimeUnit unit) throws PulsarClientException { + checkArgument(delayTime >= 0, "The delay time must not be negative."); reconsumeLater(message, null, delayTime, unit); } @@ -562,6 +564,7 @@ public CompletableFuture reconsumeLaterAsync(Message message, long dela @Override public CompletableFuture reconsumeLaterAsync( Message message, Map customProperties, long delayTime, TimeUnit unit) { + checkArgument(delayTime >= 0, "The delay time must not be negative."); if (!conf.isRetryEnable()) { return FutureUtil.failedFuture(new PulsarClientException(RECONSUME_LATER_ERROR_MSG)); } @@ -598,12 +601,14 @@ public CompletableFuture acknowledgeCumulativeAsync(Message message) { @Override public CompletableFuture reconsumeLaterCumulativeAsync(Message message, long delayTime, TimeUnit unit) { + checkArgument(delayTime >= 0, "The delay time must not be negative."); return reconsumeLaterCumulativeAsync(message, null, delayTime, unit); } @Override public CompletableFuture reconsumeLaterCumulativeAsync( Message message, Map customProperties, long delayTime, TimeUnit unit) { + checkArgument(delayTime >= 0, "The delay time must not be negative."); if (!conf.isRetryEnable()) { return FutureUtil.failedFuture(new PulsarClientException(RECONSUME_LATER_ERROR_MSG)); } @@ -1276,6 +1281,10 @@ protected boolean isValidConsumerEpoch(MessageImpl message) { return true; } + protected boolean isSingleMessageAcked(BitSetRecyclable ackBitSet, int batchIndex) { + return ackBitSet != null && !ackBitSet.get(batchIndex); + } + public boolean hasBatchReceiveTimeout() { return batchReceiveTimeout != null; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java index f644c6a18398f..895273a90c050 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java @@ -497,6 +497,7 @@ public ConsumerBuilder enableBatchIndexAcknowledgment(boolean batchIndexAckno @Override public ConsumerBuilder expireTimeOfIncompleteChunkedMessage(long duration, TimeUnit unit) { + checkArgument(duration >= 0, "expired time of incomplete chunk message must not be negative"); conf.setExpireTimeOfIncompleteChunkedMessageMillis(unit.toMillis(duration)); return this; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index b43cd79959c00..5619837757363 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -1192,7 +1192,7 @@ protected MessageImpl newSingleMessage(final int index, return null; } - if (ackBitSet != null && !ackBitSet.get(index)) { + if (isSingleMessageAcked(ackBitSet, index)) { return null; } @@ -1643,7 +1643,14 @@ void receiveIndividualMessagesFromBatch(BrokerEntryMetadata brokerEntryMetadata, singleMessageMetadata, uncompressedPayload, batchMessage, schema, true, ackBitSet, ackSetInMessageId, redeliveryCount, consumerEpoch); if (message == null) { - skippedMessages++; + // If it is not in ackBitSet, it means Broker does not want to deliver it to the client, and + // did not decrease the permits in the broker-side. + // So do not acquire more permits for this message. + // Why not skip this single message in the first line of for-loop block? We need call + // "newSingleMessage" to move "payload.readerIndex" to a correct value to get the correct data. + if (!isSingleMessageAcked(ackBitSet, i)) { + skippedMessages++; + } continue; } if (possibleToDeadLetter != null) { @@ -2172,7 +2179,8 @@ private void initDeadLetterProducerIfNeeded() { ((ProducerBuilderImpl) client.newProducer(Schema.AUTO_PRODUCE_BYTES(schema))) .initialSubscriptionName(this.deadLetterPolicy.getInitialSubscriptionName()) .topic(this.deadLetterPolicy.getDeadLetterTopic()) - .producerName(String.format("%s-%s-DLQ", this.topicName, this.subscription)) + .producerName(String.format("%s-%s-%s-DLQ", this.topicName, this.subscription, + this.consumerName)) .blockIfQueueFull(false) .enableBatching(false) .enableChunking(true) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java index 080d328e3f02c..9d30108ec7a1d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java @@ -236,6 +236,7 @@ public ControlledClusterFailoverBuilder urlProviderHeader(Map he @Override public ControlledClusterFailoverBuilder checkInterval(long interval, @NonNull TimeUnit timeUnit) { + checkArgument(interval >= 0, "The check interval time must not be negative."); this.interval = timeUnit.toMillis(interval); return this; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index 6ba3aaaaa4603..d18af475d6167 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -49,6 +49,7 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; +import javax.annotation.Nullable; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.client.api.BatchReceivePolicy; import org.apache.pulsar.client.api.Consumer; @@ -101,7 +102,8 @@ public class MultiTopicsConsumerImpl extends ConsumerBase { private final MultiTopicConsumerStatsRecorderImpl stats; private final ConsumerConfigurationData internalConfig; - private volatile MessageIdAdv startMessageId; + private final MessageIdAdv startMessageId; + private volatile boolean duringSeek = false; private final long startMessageRollbackDurationInSec; MultiTopicsConsumerImpl(PulsarClientImpl client, ConsumerConfigurationData conf, ExecutorProvider executorProvider, CompletableFuture> subscribeFuture, Schema schema, @@ -235,6 +237,10 @@ private void startReceivingMessages(List> newConsumers) { } private void receiveMessageFromConsumer(ConsumerImpl consumer, boolean batchReceive) { + if (duringSeek) { + log.info("[{}] Pause receiving messages for topic {} due to seek", subscription, consumer.getTopic()); + return; + } CompletableFuture>> messagesFuture; if (batchReceive) { messagesFuture = consumer.batchReceiveAsync().thenApply(msgs -> ((MessagesImpl) msgs).getMessageList()); @@ -252,8 +258,12 @@ private void receiveMessageFromConsumer(ConsumerImpl consumer, boolean batchR } // Process the message, add to the queue and trigger listener or async callback messages.forEach(msg -> { - if (isValidConsumerEpoch((MessageImpl) msg)) { + final boolean skipDueToSeek = duringSeek; + if (isValidConsumerEpoch((MessageImpl) msg) && !skipDueToSeek) { messageReceived(consumer, msg); + } else if (skipDueToSeek) { + log.info("[{}] [{}] Skip processing message {} received during seek", topic, subscription, + msg.getMessageId()); } }); @@ -748,17 +758,12 @@ public void seek(Function function) throws PulsarClientException @Override public CompletableFuture seekAsync(Function function) { - List> futures = new ArrayList<>(consumers.size()); - consumers.values().forEach(consumer -> futures.add(consumer.seekAsync(function))); - unAckedMessageTracker.clear(); - incomingMessages.clear(); - resetIncomingMessageSize(); - return FutureUtil.waitForAll(futures); + return seekAllAsync(consumer -> consumer.seekAsync(function)); } @Override public CompletableFuture seekAsync(MessageId messageId) { - final Consumer internalConsumer; + final ConsumerImpl internalConsumer; if (messageId instanceof TopicMessageId) { TopicMessageId topicMessageId = (TopicMessageId) messageId; internalConsumer = consumers.get(topicMessageId.getOwnerTopic()); @@ -775,25 +780,46 @@ public CompletableFuture seekAsync(MessageId messageId) { ); } - final CompletableFuture seekFuture; if (internalConsumer == null) { - List> futures = new ArrayList<>(consumers.size()); - consumers.values().forEach(consumerImpl -> futures.add(consumerImpl.seekAsync(messageId))); - seekFuture = FutureUtil.waitForAll(futures); + return seekAllAsync(consumer -> consumer.seekAsync(messageId)); } else { - seekFuture = internalConsumer.seekAsync(messageId); + return seekAsyncInternal(Collections.singleton(internalConsumer), __ -> __.seekAsync(messageId)); } + } + @Override + public CompletableFuture seekAsync(long timestamp) { + return seekAllAsync(consumer -> consumer.seekAsync(timestamp)); + } + + private CompletableFuture seekAsyncInternal(Collection> consumers, + Function, CompletableFuture> seekFunc) { + beforeSeek(); + final CompletableFuture future = new CompletableFuture<>(); + FutureUtil.waitForAll(consumers.stream().map(seekFunc).collect(Collectors.toList())) + .whenComplete((__, e) -> afterSeek(future, e)); + return future; + } + + private CompletableFuture seekAllAsync(Function, CompletableFuture> seekFunc) { + return seekAsyncInternal(consumers.values(), seekFunc); + } + + private void beforeSeek() { + duringSeek = true; unAckedMessageTracker.clear(); clearIncomingMessages(); - return seekFuture; } - @Override - public CompletableFuture seekAsync(long timestamp) { - List> futures = new ArrayList<>(consumers.size()); - consumers.values().forEach(consumer -> futures.add(consumer.seekAsync(timestamp))); - return FutureUtil.waitForAll(futures); + private void afterSeek(CompletableFuture seekFuture, @Nullable Throwable throwable) { + duringSeek = false; + log.info("[{}] Resume receiving messages for {} since seek is done", subscription, consumers.keySet()); + startReceivingMessages(new ArrayList<>(consumers.values())); + if (throwable == null) { + seekFuture.complete(null); + } else { + seekFuture.completeExceptionally(throwable); + } } @Override @@ -903,7 +929,10 @@ private void removeTopic(String topic) { } } - // subscribe one more given topic + /*** + * Subscribe one more given topic. + * @param topicName topic name without the partition suffix. + */ public CompletableFuture subscribeAsync(String topicName, boolean createTopicIfDoesNotExist) { TopicName topicNameInstance = getTopicName(topicName); if (topicNameInstance == null) { @@ -999,13 +1028,13 @@ CompletableFuture subscribeAsync(String topicName, int numberPartitions) { private void subscribeTopicPartitions(CompletableFuture subscribeResult, String topicName, int numPartitions, boolean createIfDoesNotExist) { - client.preProcessSchemaBeforeSubscribe(client, schema, topicName).whenComplete((schema, cause) -> { - if (null == cause) { - doSubscribeTopicPartitions(schema, subscribeResult, topicName, numPartitions, createIfDoesNotExist); - } else { - subscribeResult.completeExceptionally(cause); - } - }); + client.preProcessSchemaBeforeSubscribe(client, schema, topicName) + .thenAccept(schema -> { + doSubscribeTopicPartitions(schema, subscribeResult, topicName, numPartitions, createIfDoesNotExist); + }).exceptionally(cause -> { + subscribeResult.completeExceptionally(cause); + return null; + }); } private void doSubscribeTopicPartitions(Schema schema, @@ -1225,7 +1254,10 @@ public CompletableFuture unsubscribeAsync(String topicName) { return unsubscribeFuture; } - // Remove a consumer for a topic + /*** + * Remove a consumer for a topic. + * @param topicName topic name contains the partition suffix. + */ public CompletableFuture removeConsumerAsync(String topicName) { checkArgument(TopicName.isValid(topicName), "Invalid topic name:" + topicName); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java index c6ea6216cc1f4..4d179f7d914c2 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java @@ -31,6 +31,7 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.pulsar.client.api.Consumer; @@ -50,11 +51,25 @@ public class PatternMultiTopicsConsumerImpl extends MultiTopicsConsumerImpl watcherFuture; + private final CompletableFuture watcherFuture = new CompletableFuture<>(); protected NamespaceName namespaceName; + + /** + * There is two task to re-check topic changes, the both tasks will not be take affects at the same time. + * 1. {@link #recheckTopicsChangeAfterReconnect}: it will be called after the {@link TopicListWatcher} reconnected + * if you enabled {@link TopicListWatcher}. This backoff used to do a retry if + * {@link #recheckTopicsChangeAfterReconnect} is failed. + * 2. {@link #run} A scheduled task to trigger re-check topic changes, it will be used if you disabled + * {@link TopicListWatcher}. + */ + private final Backoff recheckPatternTaskBackoff; + private final AtomicInteger recheckPatternEpoch = new AtomicInteger(); private volatile Timeout recheckPatternTimeout = null; private volatile String topicsHash; + /*** + * @param topicsPattern The regexp for the topic name(not contains partition suffix). + */ public PatternMultiTopicsConsumerImpl(Pattern topicsPattern, String topicsHash, PulsarClientImpl client, @@ -69,6 +84,11 @@ public PatternMultiTopicsConsumerImpl(Pattern topicsPattern, this.topicsPattern = topicsPattern; this.topicsHash = topicsHash; this.subscriptionMode = subscriptionMode; + this.recheckPatternTaskBackoff = new BackoffBuilder() + .setInitialTime(client.getConfiguration().getInitialBackoffIntervalNanos(), TimeUnit.NANOSECONDS) + .setMax(client.getConfiguration().getMaxBackoffIntervalNanos(), TimeUnit.NANOSECONDS) + .setMandatoryStop(0, TimeUnit.SECONDS) + .create(); if (this.namespaceName == null) { this.namespaceName = getNameSpaceFromPattern(topicsPattern); @@ -78,11 +98,10 @@ public PatternMultiTopicsConsumerImpl(Pattern topicsPattern, this.topicsChangeListener = new PatternTopicsChangedListener(); this.recheckPatternTimeout = client.timer() .newTimeout(this, Math.max(1, conf.getPatternAutoDiscoveryPeriod()), TimeUnit.SECONDS); - this.watcherFuture = new CompletableFuture<>(); if (subscriptionMode == Mode.PERSISTENT) { long watcherId = client.newTopicListWatcherId(); new TopicListWatcher(topicsChangeListener, client, topicsPattern, watcherId, - namespaceName, topicsHash, watcherFuture); + namespaceName, topicsHash, watcherFuture, () -> recheckTopicsChangeAfterReconnect()); watcherFuture .thenAccept(__ -> recheckPatternTimeout.cancel()) .exceptionally(ex -> { @@ -99,40 +118,75 @@ public static NamespaceName getNameSpaceFromPattern(Pattern pattern) { return TopicName.get(pattern.pattern()).getNamespaceObject(); } + /** + * This method will be called after the {@link TopicListWatcher} reconnected after enabled {@link TopicListWatcher}. + */ + private void recheckTopicsChangeAfterReconnect() { + // Skip if closed or the task has been cancelled. + if (getState() == State.Closing || getState() == State.Closed) { + return; + } + // Do check. + recheckTopicsChange().whenComplete((ignore, ex) -> { + if (ex != null) { + log.warn("[{}] Failed to recheck topics change: {}", topic, ex.getMessage()); + long delayMs = recheckPatternTaskBackoff.next(); + client.timer().newTimeout(timeout -> { + recheckTopicsChangeAfterReconnect(); + }, delayMs, TimeUnit.MILLISECONDS); + } else { + recheckPatternTaskBackoff.reset(); + } + }); + } + // TimerTask to recheck topics change, and trigger subscribe/unsubscribe based on the change. @Override public void run(Timeout timeout) throws Exception { if (timeout.isCancelled()) { return; } - client.getLookup().getTopicsUnderNamespace(namespaceName, subscriptionMode, topicsPattern.pattern(), topicsHash) - .thenCompose(getTopicsResult -> { + recheckTopicsChange().exceptionally(ex -> { + log.warn("[{}] Failed to recheck topics change: {}", topic, ex.getMessage()); + return null; + }).thenAccept(__ -> { + // schedule the next re-check task + this.recheckPatternTimeout = client.timer() + .newTimeout(PatternMultiTopicsConsumerImpl.this, + Math.max(1, conf.getPatternAutoDiscoveryPeriod()), TimeUnit.SECONDS); + }); + } - if (log.isDebugEnabled()) { - log.debug("Get topics under namespace {}, topics.size: {}, topicsHash: {}, filtered: {}", - namespaceName, getTopicsResult.getTopics().size(), getTopicsResult.getTopicsHash(), - getTopicsResult.isFiltered()); - getTopicsResult.getTopics().forEach(topicName -> - log.debug("Get topics under namespace {}, topic: {}", namespaceName, topicName)); - } + private CompletableFuture recheckTopicsChange() { + String pattern = topicsPattern.pattern(); + final int epoch = recheckPatternEpoch.incrementAndGet(); + return client.getLookup().getTopicsUnderNamespace(namespaceName, subscriptionMode, pattern, topicsHash) + .thenCompose(getTopicsResult -> { + // If "recheckTopicsChange" has been called more than one times, only make the last one take affects. + // Use "synchronized (recheckPatternTaskBackoff)" instead of + // `synchronized(PatternMultiTopicsConsumerImpl.this)` to avoid locking in a wider range. + synchronized (recheckPatternTaskBackoff) { + if (recheckPatternEpoch.get() > epoch) { + return CompletableFuture.completedFuture(null); + } + if (log.isDebugEnabled()) { + log.debug("Get topics under namespace {}, topics.size: {}, topicsHash: {}, filtered: {}", + namespaceName, getTopicsResult.getTopics().size(), getTopicsResult.getTopicsHash(), + getTopicsResult.isFiltered()); + getTopicsResult.getTopics().forEach(topicName -> + log.debug("Get topics under namespace {}, topic: {}", namespaceName, topicName)); + } - final List oldTopics = new ArrayList<>(getPartitionedTopics()); - for (String partition : getPartitions()) { - TopicName topicName = TopicName.get(partition); - if (!topicName.isPartitioned() || !oldTopics.contains(topicName.getPartitionedTopicName())) { - oldTopics.add(partition); + final List oldTopics = new ArrayList<>(getPartitionedTopics()); + for (String partition : getPartitions()) { + TopicName topicName = TopicName.get(partition); + if (!topicName.isPartitioned() || !oldTopics.contains(topicName.getPartitionedTopicName())) { + oldTopics.add(partition); + } } + return updateSubscriptions(topicsPattern, this::setTopicsHash, getTopicsResult, + topicsChangeListener, oldTopics); } - return updateSubscriptions(topicsPattern, this::setTopicsHash, getTopicsResult, - topicsChangeListener, oldTopics); - }).exceptionally(ex -> { - log.warn("[{}] Failed to recheck topics change: {}", topic, ex.getMessage()); - return null; - }).thenAccept(__ -> { - // schedule the next re-check task - this.recheckPatternTimeout = client.timer() - .newTimeout(PatternMultiTopicsConsumerImpl.this, - Math.max(1, conf.getPatternAutoDiscoveryPeriod()), TimeUnit.SECONDS); }); } @@ -169,14 +223,26 @@ void setTopicsHash(String topicsHash) { } interface TopicsChangedListener { - // unsubscribe and delete ConsumerImpl in the `consumers` map in `MultiTopicsConsumerImpl` based on added topics + /*** + * unsubscribe and delete {@link ConsumerImpl} in the {@link MultiTopicsConsumerImpl#consumers} map in + * {@link MultiTopicsConsumerImpl}. + * @param removedTopics topic names removed(contains the partition suffix). + */ CompletableFuture onTopicsRemoved(Collection removedTopics); - // subscribe and create a list of new ConsumerImpl, added them to the `consumers` map in - // `MultiTopicsConsumerImpl`. + + /*** + * subscribe and create a list of new {@link ConsumerImpl}, added them to the + * {@link MultiTopicsConsumerImpl#consumers} map in {@link MultiTopicsConsumerImpl}. + * @param addedTopics topic names added(contains the partition suffix). + */ CompletableFuture onTopicsAdded(Collection addedTopics); } private class PatternTopicsChangedListener implements TopicsChangedListener { + + /** + * {@inheritDoc} + */ @Override public CompletableFuture onTopicsRemoved(Collection removedTopics) { CompletableFuture removeFuture = new CompletableFuture<>(); @@ -198,6 +264,9 @@ public CompletableFuture onTopicsRemoved(Collection removedTopics) return removeFuture; } + /** + * {@inheritDoc} + */ @Override public CompletableFuture onTopicsAdded(Collection addedTopics) { CompletableFuture addFuture = new CompletableFuture<>(); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 2763da524cd58..4908d10f330b3 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -1983,6 +1983,11 @@ String getHandlerName() { return producerName; } + @VisibleForTesting + void triggerSendTimer() throws Exception { + run(sendTimeout); + } + /** * Process sendTimeout events. */ @@ -2001,7 +2006,8 @@ public void run(Timeout timeout) throws Exception { } OpSendMsg firstMsg = pendingMessages.peek(); - if (firstMsg == null && (batchMessageContainer == null || batchMessageContainer.isEmpty())) { + if (firstMsg == null && (batchMessageContainer == null || batchMessageContainer.isEmpty() + || batchMessageContainer.getFirstAddedTimestamp() == 0L)) { // If there are no pending messages, reset the timeout to the configured value. timeToWaitMs = conf.getSendTimeoutMs(); } else { @@ -2011,7 +2017,7 @@ public void run(Timeout timeout) throws Exception { } else { // Because we don't flush batch messages while disconnected, we consider them "createdAt" when // they would have otherwise been flushed. - createdAt = lastBatchSendNanoTime + createdAt = batchMessageContainer.getFirstAddedTimestamp() + TimeUnit.MICROSECONDS.toNanos(conf.getBatchingMaxPublishDelayMicros()); } // If there is at least one message, calculate the diff between the message timeout and the elapsed diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderBuilderImpl.java index 2860cda0ceef1..fd01cef9a216f 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderBuilderImpl.java @@ -274,6 +274,7 @@ public ReaderBuilder autoAckOldestChunkedMessageOnQueueFull(boolean autoAckOl @Override public ReaderBuilder expireTimeOfIncompleteChunkedMessage(long duration, TimeUnit unit) { + checkArgument(duration >= 0, "The expired time must not be negative."); conf.setExpireTimeOfIncompleteChunkedMessageMillis(unit.toMillis(duration)); return this; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java index 151c96d96aa40..64abd6d811b8e 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java @@ -304,8 +304,14 @@ private void readTailMessages(Reader reader) { log.error("Reader {} was closed while reading tail messages.", reader.getTopic(), ex); } else { + // Retrying on the other exceptions such as NotConnectedException + try { + Thread.sleep(50); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } log.warn("Reader {} was interrupted while reading tail messages. " - + "Retrying..", reader.getTopic(), ex); + + "Retrying..", reader.getTopic(), ex); readTailMessages(reader); } return null; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java index 2ce784dbaac04..86adf69f06e0f 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java @@ -56,11 +56,17 @@ public class TopicListWatcher extends HandlerState implements ConnectionHandler. private final List previousExceptions = new CopyOnWriteArrayList<>(); private final AtomicReference clientCnxUsedForWatcherRegistration = new AtomicReference<>(); + private final Runnable recheckTopicsChangeAfterReconnect; + + /*** + * @param topicsPattern The regexp for the topic name(not contains partition suffix). + */ public TopicListWatcher(PatternMultiTopicsConsumerImpl.TopicsChangedListener topicsChangeListener, PulsarClientImpl client, Pattern topicsPattern, long watcherId, NamespaceName namespace, String topicsHash, - CompletableFuture watcherFuture) { + CompletableFuture watcherFuture, + Runnable recheckTopicsChangeAfterReconnect) { super(client, topicsPattern.pattern()); this.topicsChangeListener = topicsChangeListener; this.name = "Watcher(" + topicsPattern + ")"; @@ -77,6 +83,7 @@ public TopicListWatcher(PatternMultiTopicsConsumerImpl.TopicsChangedListener top this.namespace = namespace; this.topicsHash = topicsHash; this.watcherFuture = watcherFuture; + this.recheckTopicsChangeAfterReconnect = recheckTopicsChangeAfterReconnect; connectionHandler.grabCnx(); } @@ -138,9 +145,9 @@ public CompletableFuture connectionOpened(ClientCnx cnx) { return; } } - this.connectionHandler.resetBackoff(); + recheckTopicsChangeAfterReconnect.run(); watcherFuture.complete(this); future.complete(null); }).exceptionally((e) -> { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java index 0b5174a015118..4ea6472b9d8b2 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.impl; +import static com.google.common.base.Preconditions.checkArgument; import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.ByteBuf; import io.netty.util.Recycler; @@ -205,6 +206,7 @@ private void failPendingRequest() { } public CompletableFuture newTransactionAsync(long timeout, TimeUnit unit) { + checkArgument(timeout >= 0, "The timeout must not be negative."); if (LOG.isDebugEnabled()) { LOG.debug("New transaction with timeout in ms {}", unit.toMillis(timeout)); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java index 026f8a1e69e0b..895949fdf32cc 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java @@ -212,6 +212,7 @@ public TypedMessageBuilder disableReplication() { @Override public TypedMessageBuilder deliverAfter(long delay, TimeUnit unit) { + checkArgument(delay >= 0, "The delay time must not be negative."); return deliverAt(System.currentTimeMillis() + unit.toMillis(delay)); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java index 8760926792cd7..3ae0e977d13c4 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java @@ -65,7 +65,7 @@ public class ConsumerConfigurationData implements Serializable, Cloneable { @ApiModelProperty( name = "topicsPattern", - value = "Topic pattern" + value = "The regexp for the topic name(not contains partition suffix)." ) private Pattern topicsPattern; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBuilderImpl.java index 0ebfb91e62da7..bb0c4968fc7d4 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBuilderImpl.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.impl.transaction; +import static com.google.common.base.Preconditions.checkArgument; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; @@ -45,6 +46,7 @@ public TransactionBuilderImpl(PulsarClientImpl client, TransactionCoordinatorCli @Override public TransactionBuilder withTransactionTimeout(long txnTimeout, TimeUnit timeoutUnit) { + checkArgument(txnTimeout >= 0, "The txn timeout must not be negative."); this.txnTimeout = txnTimeout; this.timeUnit = timeoutUnit; return this; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/util/ObjectCache.java b/pulsar-client/src/main/java/org/apache/pulsar/client/util/ObjectCache.java index dc057ffe32daf..cf0620edf98df 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/util/ObjectCache.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/util/ObjectCache.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.util; +import static com.google.common.base.Preconditions.checkArgument; import java.time.Clock; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -33,6 +34,7 @@ public class ObjectCache implements Supplier { public ObjectCache(Supplier supplier, long cacheDuration, TimeUnit unit) { this(supplier, cacheDuration, unit, Clock.systemUTC()); + checkArgument(cacheDuration >= 0, "The cache duration must not be negative."); } ObjectCache(Supplier supplier, long cacheDuration, TimeUnit unit, Clock clock) { diff --git a/pulsar-client/src/main/resources/findbugsExclude.xml b/pulsar-client/src/main/resources/findbugsExclude.xml index 0e05d20cb9bb4..478ed1bff49d6 100644 --- a/pulsar-client/src/main/resources/findbugsExclude.xml +++ b/pulsar-client/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicListWatcherTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicListWatcherTest.java index dd75770b5688d..7e9fd601d4f67 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicListWatcherTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicListWatcherTest.java @@ -71,7 +71,7 @@ public void setup() { watcherFuture = new CompletableFuture<>(); watcher = new TopicListWatcher(listener, client, Pattern.compile(topic), 7, - NamespaceName.get("tenant/ns"), null, watcherFuture); + NamespaceName.get("tenant/ns"), null, watcherFuture, () -> {}); } @Test diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index e67116eed7a2b..b2eed1cd470a0 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - pulsar-common Pulsar Common Common libraries needed by client/broker/tools - ${project.groupId} pulsar-client-api ${project.version} - ${project.groupId} pulsar-client-admin-api ${project.version} - io.swagger swagger-annotations provided - org.slf4j slf4j-api - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.module jackson-module-parameter-names - com.fasterxml.jackson.datatype jackson-datatype-jsr310 - com.fasterxml.jackson.datatype jackson-datatype-jdk8 - com.google.guava guava - io.netty netty-handler - io.netty netty-resolver-dns - io.netty netty-transport-native-epoll linux-x86_64 - io.netty netty-transport-native-unix-common linux-x86_64 - org.apache.bookkeeper bookkeeper-common-allocator @@ -115,17 +98,14 @@ - org.apache.bookkeeper cpu-affinity - io.airlift aircompressor - org.apache.bookkeeper circe-checksum @@ -141,66 +121,54 @@ - io.netty netty-tcnative-boringssl-static - io.netty.incubator netty-incubator-transport-classes-io_uring - io.netty.incubator netty-incubator-transport-native-io_uring linux-x86_64 - io.netty.incubator netty-incubator-transport-native-io_uring linux-aarch_64 - io.netty netty-codec-haproxy - org.apache.commons commons-lang3 - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - javax.ws.rs javax.ws.rs-api - commons-io commons-io - com.github.spotbugs spotbugs-annotations provided true - com.google.protobuf - protobuf-java + protobuf-java - org.bouncycastle @@ -208,38 +176,32 @@ ${bouncycastle.bc-fips.version} test - org.lz4 lz4-java 1.5.0 test - com.github.luben zstd-jni test - org.xerial.snappy snappy-java test - com.google.code.gson gson - org.awaitility awaitility test - @@ -249,7 +211,6 @@ ${pulsar.client.compiler.release} - org.gaul modernizer-maven-plugin @@ -267,7 +228,6 @@ - com.github.splunk.lightproto lightproto-maven-plugin @@ -280,7 +240,6 @@ - pl.project13.maven git-commit-id-plugin @@ -305,7 +264,6 @@ - org.codehaus.mojo templating-maven-plugin diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoader.java b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoader.java index 620e1156d3555..9736d8b47ef71 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoader.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoader.java @@ -154,6 +154,11 @@ public NarClassLoader run() { }); } + public static List getClasspathFromArchive(File narPath, String narExtractionDirectory) throws IOException { + File unpacked = NarUnpacker.unpackNar(narPath, getNarExtractionDirectory(narExtractionDirectory)); + return getClassPathEntries(unpacked); + } + private static File getNarExtractionDirectory(String configuredDirectory) { return new File(configuredDirectory + "/" + TMP_DIR_PREFIX); } @@ -164,16 +169,11 @@ private static File getNarExtractionDirectory(String configuredDirectory) { * @param narWorkingDirectory * directory to explode nar contents to * @param parent - * @throws IllegalArgumentException - * if the NAR is missing the Java Services API file for FlowFileProcessor implementations. - * @throws ClassNotFoundException - * if any of the FlowFileProcessor implementations defined by the Java Services API cannot be - * loaded. * @throws IOException * if an error occurs while loading the NAR. */ private NarClassLoader(final File narWorkingDirectory, Set additionalJars, ClassLoader parent) - throws ClassNotFoundException, IOException { + throws IOException { super(new URL[0], parent); this.narWorkingDirectory = narWorkingDirectory; @@ -238,22 +238,31 @@ public List getServiceImplementation(String serviceName) throws IOExcept * if the URL list could not be updated. */ private void updateClasspath(File root) throws IOException { - addURL(root.toURI().toURL()); // for compiled classes, META-INF/, etc. + getClassPathEntries(root).forEach(f -> { + try { + addURL(f.toURI().toURL()); + } catch (IOException e) { + log.error("Failed to add entry to classpath: {}", f, e); + } + }); + } + static List getClassPathEntries(File root) { + List classPathEntries = new ArrayList<>(); + classPathEntries.add(root); File dependencies = new File(root, "META-INF/bundled-dependencies"); if (!dependencies.isDirectory()) { - log.warn("{} does not contain META-INF/bundled-dependencies!", narWorkingDirectory); + log.warn("{} does not contain META-INF/bundled-dependencies!", root); } - addURL(dependencies.toURI().toURL()); + classPathEntries.add(dependencies); if (dependencies.isDirectory()) { final File[] jarFiles = dependencies.listFiles(JAR_FILTER); if (jarFiles != null) { Arrays.sort(jarFiles, Comparator.comparing(File::getName)); - for (File libJar : jarFiles) { - addURL(libJar.toURI().toURL()); - } + classPathEntries.addAll(Arrays.asList(jarFiles)); } } + return classPathEntries; } @Override diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java index 9bd5bc48df819..1e34c3e4fe706 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java @@ -32,13 +32,14 @@ import java.io.RandomAccessFile; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; +import java.nio.file.Path; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.Base64; import java.util.Enumeration; import java.util.concurrent.ConcurrentHashMap; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; import lombok.extern.slf4j.Slf4j; /** @@ -113,18 +114,24 @@ static File doUnpackNar(final File nar, final File baseWorkingDirectory, Runnabl * if the NAR could not be unpacked. */ private static void unpack(final File nar, final File workingDirectory) throws IOException { - try (JarFile jarFile = new JarFile(nar)) { - Enumeration jarEntries = jarFile.entries(); - while (jarEntries.hasMoreElements()) { - JarEntry jarEntry = jarEntries.nextElement(); - String name = jarEntry.getName(); - File f = new File(workingDirectory, name); - if (jarEntry.isDirectory()) { + Path workingDirectoryPath = workingDirectory.toPath().normalize(); + try (ZipFile zipFile = new ZipFile(nar)) { + Enumeration zipEntries = zipFile.entries(); + while (zipEntries.hasMoreElements()) { + ZipEntry zipEntry = zipEntries.nextElement(); + String name = zipEntry.getName(); + Path targetFilePath = workingDirectoryPath.resolve(name).normalize(); + if (!targetFilePath.startsWith(workingDirectoryPath)) { + log.error("Invalid zip file with entry '{}'", name); + throw new IOException("Invalid zip file. Aborting unpacking."); + } + File f = targetFilePath.toFile(); + if (zipEntry.isDirectory()) { FileUtils.ensureDirectoryExistAndCanReadAndWrite(f); } else { // The directory entry might appear after the file entry FileUtils.ensureDirectoryExistAndCanReadAndWrite(f.getParentFile()); - makeFile(jarFile.getInputStream(jarEntry), f); + makeFile(zipFile.getInputStream(zipEntry), f); } } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java index 34d47e2836bb2..65674af0ae14e 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java @@ -1585,6 +1585,9 @@ public static BaseCommand newWatchTopicList( return cmd; } + /*** + * @param topics topic names which are matching, the topic name contains the partition suffix. + */ public static BaseCommand newWatchTopicListSuccess(long requestId, long watcherId, String topicsHash, List topics) { BaseCommand cmd = localCmd(Type.WATCH_TOPIC_LIST_SUCCESS); @@ -1600,6 +1603,10 @@ public static BaseCommand newWatchTopicListSuccess(long requestId, long watcherI return cmd; } + /** + * @param deletedTopics topic names deleted(contains the partition suffix). + * @param newTopics topics names added(contains the partition suffix). + */ public static BaseCommand newWatchTopicUpdate(long watcherId, List newTopics, List deletedTopics, String topicsHash) { BaseCommand cmd = localCmd(Type.WATCH_TOPIC_UPDATE); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicList.java b/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicList.java index 250cea217ee5f..4c0a8d500b703 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicList.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicList.java @@ -47,13 +47,16 @@ public static List filterTopics(List original, String regex) { } public static List filterTopics(List original, Pattern topicsPattern) { - final Pattern shortenedTopicsPattern = topicsPattern.toString().contains(SCHEME_SEPARATOR) - ? Pattern.compile(SCHEME_SEPARATOR_PATTERN.split(topicsPattern.toString())[1]) : topicsPattern; + final Pattern shortenedTopicsPattern = Pattern.compile(removeTopicDomainScheme(topicsPattern.toString())); return original.stream() .map(TopicName::get) + .filter(topicName -> { + String partitionedTopicName = topicName.getPartitionedTopicName(); + String removedScheme = SCHEME_SEPARATOR_PATTERN.split(partitionedTopicName)[1]; + return shortenedTopicsPattern.matcher(removedScheme).matches(); + }) .map(TopicName::toString) - .filter(topic -> shortenedTopicsPattern.matcher(SCHEME_SEPARATOR_PATTERN.split(topic)[1]).matches()) .collect(Collectors.toList()); } @@ -78,4 +81,16 @@ public static Set minus(Collection list1, Collection lis s1.removeAll(list2); return s1; } + + private static String removeTopicDomainScheme(String originalRegexp) { + if (!originalRegexp.toString().contains(SCHEME_SEPARATOR)) { + return originalRegexp; + } + String removedTopicDomain = SCHEME_SEPARATOR_PATTERN.split(originalRegexp.toString())[1]; + if (originalRegexp.contains("^")) { + return String.format("^%s", removedTopicDomain); + } else { + return removedTopicDomain; + } + } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/BitSetRecyclable.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/BitSetRecyclable.java index 12ce7eb74c72b..b801d5f2b05a1 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/BitSetRecyclable.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/BitSetRecyclable.java @@ -216,6 +216,14 @@ public static BitSetRecyclable valueOf(byte[] bytes) { return BitSetRecyclable.valueOf(ByteBuffer.wrap(bytes)); } + /** + * Copy a BitSetRecyclable. + */ + public static BitSetRecyclable valueOf(BitSetRecyclable src) { + // The internal implementation will do the array-copy. + return valueOf(src.words); + } + /** * Returns a new bit set containing all the bits in the given byte * buffer between its position and limit. diff --git a/pulsar-common/src/main/resources/findbugsExclude.xml b/pulsar-common/src/main/resources/findbugsExclude.xml index df161c4b621a7..8216948c12408 100644 --- a/pulsar-common/src/main/resources/findbugsExclude.xml +++ b/pulsar-common/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pulsar-config-validation/pom.xml b/pulsar-config-validation/pom.xml index e09a488f2cb16..ccab028190f09 100644 --- a/pulsar-config-validation/pom.xml +++ b/pulsar-config-validation/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar - pulsar - 3.2.0-SNAPSHOT - .. - - - pulsar-config-validation - Annotation based config validation for Pulsar - - - - org.slf4j - slf4j-api - - - - - - - org.gaul - modernizer-maven-plugin - - true - 17 - - - - modernizer - verify - - modernizer - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - - \ No newline at end of file + + 4.0.0 + + io.streamnative + pulsar + 3.2.0-SNAPSHOT + .. + + pulsar-config-validation + Annotation based config validation for Pulsar + + + org.slf4j + slf4j-api + + + + + + org.gaul + modernizer-maven-plugin + + true + 17 + + + + modernizer + verify + + modernizer + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + + diff --git a/pulsar-docs-tools/pom.xml b/pulsar-docs-tools/pom.xml index e05d0ee0ea70e..843739c6edb36 100644 --- a/pulsar-docs-tools/pom.xml +++ b/pulsar-docs-tools/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar - pulsar - 3.2.0-SNAPSHOT - - - pulsar-docs-tools - Pulsar Documentation Generators - - - - io.swagger - swagger-annotations - - - io.swagger - swagger-core - - - com.beust - jcommander - - - + + 4.0.0 + + io.streamnative + pulsar + 3.2.0-SNAPSHOT + + pulsar-docs-tools + Pulsar Documentation Generators + + + io.swagger + swagger-annotations + + + io.swagger + swagger-core + + + com.beust + jcommander + + diff --git a/pulsar-functions/api-java/pom.xml b/pulsar-functions/api-java/pom.xml index 0cb2a323c8b83..fa58820396cf7 100644 --- a/pulsar-functions/api-java/pom.xml +++ b/pulsar-functions/api-java/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-functions 3.2.0-SNAPSHOT - pulsar-functions-api Pulsar Functions :: API - - org.slf4j slf4j-api - net.jodah typetools @@ -48,16 +44,13 @@ ${project.version} compile - ${project.groupId} pulsar-client-admin-api ${project.version} compile - - @@ -81,16 +74,15 @@ org.apache.maven.plugins maven-checkstyle-plugin - - checkstyle - verify - - check - - + + checkstyle + verify + + check + + - diff --git a/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/state/ByteBufferStateStore.java b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/state/ByteBufferStateStore.java index 8dbd7b322a5ee..d938fe0c82b7d 100644 --- a/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/state/ByteBufferStateStore.java +++ b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/state/ByteBufferStateStore.java @@ -73,4 +73,31 @@ public interface ByteBufferStateStore extends StateStore { */ CompletableFuture getAsync(String key); + /** + * Retrieve the StateValue for the key. + * + * @param key name of the key + * @return the StateValue. + */ + default StateValue getStateValue(String key) { + return getStateValueAsync(key).join(); + } + + /** + * Retrieve the StateValue for the key, but don't wait for the operation to be completed. + * + * @param key name of the key + * @return the StateValue. + */ + default CompletableFuture getStateValueAsync(String key) { + return getAsync(key).thenApply(val -> { + if (val != null && val.remaining() >= 0) { + byte[] data = new byte[val.remaining()]; + val.get(data); + return new StateValue(data, null, null); + } else { + return null; + } + }); + } } diff --git a/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/state/StateValue.java b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/state/StateValue.java new file mode 100644 index 0000000000000..ce06b54a6e490 --- /dev/null +++ b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/state/StateValue.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.api.state; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public class StateValue { + private final byte[] value; + private final Long version; + private final Boolean isNumber; +} \ No newline at end of file diff --git a/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml b/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml index 9638cfcca8da9..4eb1f22f5c8b7 100644 --- a/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-functions 3.2.0-SNAPSHOT - pulsar-functions-instance Pulsar Functions :: Instance - - org.apache.logging.log4j @@ -46,55 +42,46 @@ org.apache.logging.log4j log4j-core - ${project.groupId} pulsar-functions-utils ${project.version} - ${project.groupId} pulsar-metadata ${project.version} - ${project.groupId} pulsar-io-core ${project.version} - ${project.groupId} pulsar-functions-api ${project.version} - ${project.groupId} pulsar-functions-secrets ${project.version} - - + ${project.groupId} pulsar-client-admin-original ${project.version} - - + ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} pulsar-client-messagecrypto-bc ${project.parent.version} - org.apache.bookkeeper stream-storage-java-client @@ -109,24 +96,20 @@ - io.grpc grpc-stub - io.grpc grpc-all - io.perfmark perfmark-api runtime - org.apache.bookkeeper bookkeeper-common @@ -141,61 +124,50 @@ - com.yahoo.datasketches sketches-core - com.google.guava guava - com.beust jcommander - net.jodah typetools - io.prometheus simpleclient ${prometheus.version} - io.prometheus simpleclient_hotspot ${prometheus.version} - io.prometheus simpleclient_httpserver ${prometheus.version} - io.prometheus.jmx collector ${prometheus-jmx.version} - org.awaitility awaitility test - - @@ -215,7 +187,6 @@ - @@ -280,5 +251,4 @@ - diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreImpl.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreImpl.java index bf43f18b175e7..d85e4afd762ca 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreImpl.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreImpl.java @@ -28,6 +28,7 @@ import org.apache.bookkeeper.api.kv.Table; import org.apache.bookkeeper.api.kv.options.Options; import org.apache.pulsar.functions.api.StateStoreContext; +import org.apache.pulsar.functions.api.state.StateValue; import org.apache.pulsar.functions.utils.FunctionCommon; /** @@ -190,4 +191,33 @@ public ByteBuffer get(String key) { throw new RuntimeException("Failed to retrieve the state value for key '" + key + "'", e); } } + + @Override + public StateValue getStateValue(String key) { + try { + return result(getStateValueAsync(key)); + } catch (Exception e) { + throw new RuntimeException("Failed to retrieve the state value for key '" + key + "'", e); + } + } + + @Override + public CompletableFuture getStateValueAsync(String key) { + return table.getKv(Unpooled.wrappedBuffer(key.getBytes(UTF_8))).thenApply( + data -> { + try { + if (data != null && data.value() != null && data.value().readableBytes() >= 0) { + byte[] result = new byte[data.value().readableBytes()]; + data.value().readBytes(result); + return new StateValue(result, data.version(), data.isNumber()); + } + return null; + } finally { + if (data != null) { + ReferenceCountUtil.safeRelease(data); + } + } + } + ); + } } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreImpl.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreImpl.java index 50541c40ae973..bba3cea0d8f38 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreImpl.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreImpl.java @@ -22,6 +22,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import org.apache.pulsar.functions.api.StateStoreContext; +import org.apache.pulsar.functions.api.state.StateValue; import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.MetadataStore; @@ -111,6 +112,20 @@ public CompletableFuture getAsync(String key) { .orElse(null)); } + @Override + public StateValue getStateValue(String key) { + return getStateValueAsync(key).join(); + } + + @Override + public CompletableFuture getStateValueAsync(String key) { + return store.get(getPath(key)) + .thenApply(optRes -> + optRes.map(x -> + new StateValue(x.getValue(), x.getStat().getVersion(), null)) + .orElse(null)); + } + @Override public void incrCounter(String key, long amount) { incrCounterAsync(key, amount).join(); diff --git a/pulsar-functions/instance/src/main/resources/findbugsExclude.xml b/pulsar-functions/instance/src/main/resources/findbugsExclude.xml index 7fe247d2ab20a..4c04f27f66fbe 100644 --- a/pulsar-functions/instance/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/instance/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-functions 3.2.0-SNAPSHOT - pulsar-functions-api-examples-builtin Pulsar Functions :: API Examples (NAR) - ${project.groupId} @@ -37,14 +35,36 @@ ${project.version} - org.apache.nifi nifi-nar-maven-plugin + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + - diff --git a/pulsar-functions/java-examples/pom.xml b/pulsar-functions/java-examples/pom.xml index 8efd352ab2e64..f3ec2162d8d98 100644 --- a/pulsar-functions/java-examples/pom.xml +++ b/pulsar-functions/java-examples/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-functions 3.2.0-SNAPSHOT - pulsar-functions-api-examples Pulsar Functions :: API Examples - org.slf4j slf4j-api - ${project.groupId} pulsar-functions-api @@ -52,7 +49,6 @@ ${project.version} - @@ -87,5 +83,4 @@ - diff --git a/pulsar-functions/java-examples/src/main/resources/findbugsExclude.xml b/pulsar-functions/java-examples/src/main/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-functions/java-examples/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/java-examples/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml index ee439ee992e62..483d76cf7a407 100644 --- a/pulsar-functions/localrun-shaded/pom.xml +++ b/pulsar-functions/localrun-shaded/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar - pulsar-functions - 3.2.0-SNAPSHOT - .. - - - pulsar-functions-local-runner - Pulsar Functions :: Local Runner Shaded - - - - ${project.groupId} - pulsar-functions-local-runner-original - ${project.parent.version} - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - unpack - prepare-package - - unpack - - - - - org.asynchttpclient - async-http-client - ${asynchttpclient.version} - jar - true - org/asynchttpclient/config/ahc-default.properties - ${project.build.directory}/classes - - - - - - - - - maven-antrun-plugin - - - shade-ahc-properties - prepare-package - - run - - - - - - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - ${shadePluginPhase} - - shade - - - false - - - - - - - org.apache.pulsar:* - org.apache.bookkeeper:* - commons-*:* - org.apache.commons:* - com.fasterxml.jackson.*:* - io.netty:* - com.google.*:* - javax.servlet:* - org.reactivestreams:reactive-streams - org.apache.commons:* - io.swagger:* - org.yaml:snakeyaml - io.perfmark:* - io.prometheus:* - io.prometheus.jmx:* - javax.ws.rs:* - org.tukaani:xz - com.github.zafarkhaja:java-semver - net.java.dev.jna:* - org.apache.zookeeper:* - com.thoughtworks.paranamer:paranamer - jline:* - org.rocksdb:* - org.eclipse.jetty*:* - org.apache.avro:avro - com.beust:* - net.jodah:* - io.airlift:* - com.yahoo.datasketches:* - io.netty.resolver:* - - - - - org.apache.pulsar:pulsar-client-original - - ** - - - - org/bouncycastle/** - - - - - - + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + ${shadePluginPhase} + + shade + + + false + + + + + + + io.streamnative:* + org.apache.bookkeeper:* + commons-*:* + org.apache.commons:* + com.fasterxml.jackson.*:* + io.netty:* + com.google.*:* + javax.servlet:* + org.reactivestreams:reactive-streams + org.apache.commons:* + io.swagger:* + org.yaml:snakeyaml + io.perfmark:* + io.prometheus:* + io.prometheus.jmx:* + javax.ws.rs:* + org.tukaani:xz + com.github.zafarkhaja:java-semver + net.java.dev.jna:* + org.apache.zookeeper:* + com.thoughtworks.paranamer:paranamer + jline:* + org.rocksdb:* + org.eclipse.jetty*:* + org.apache.avro:avro + com.beust:* + net.jodah:* + io.airlift:* + com.yahoo.datasketches:* + io.netty.resolver:* + + + + + io.streamnative:pulsar-client-original + + ** + + + + org/bouncycastle/** + + + + + + - - com.google - org.apache.pulsar.functions.runtime.shaded.com.google - - - org.apache.jute - org.apache.pulsar.functions.runtime.shaded.org.apache.jute - - - javax.servlet - org.apache.pulsar.functions.runtime.shaded.javax.servlet - - - net.jodah - org.apache.pulsar.functions.runtime.shaded.net.jodah - - - org.lz4 - org.apache.pulsar.functions.runtime.shaded.org.lz4 - - - org.reactivestreams - org.apache.pulsar.functions.runtime.shaded.org.reactivestreams - - - org.apache.commons - org.apache.pulsar.functions.runtime.shaded.org.apache.commons - - - io.swagger - org.apache.pulsar.functions.runtime.shaded.io.swagger - - - org.yaml - org.apache.pulsar.functions.runtime.shaded.org.yaml - - - org.jctools - org.apache.pulsar.functions.runtime.shaded.org.jctools - - - com.squareup.okhttp - org.apache.pulsar.functions.runtime.shaded.com.squareup.okhttp - - - io.grpc - org.apache.pulsar.functions.runtime.shaded.io.grpc - - - io.perfmark - org.apache.pulsar.functions.runtime.shaded.io.perfmark - - - org.joda - org.apache.pulsar.functions.runtime.shaded.org.joda - - - javax.ws.rs - org.apache.pulsar.functions.runtime.shaded.javax.ws.rs - - - io.kubernetes - org.apache.pulsar.functions.runtime.shaded.io.kubernetes - - - io.opencensus - org.apache.pulsar.functions.runtime.shaded.io.opencensus - - - net.jpountz - org.apache.pulsar.functions.runtime.shaded.net.jpountz - - - commons-configuration - org.apache.pulsar.functions.runtime.shaded.commons-configuration - - - org.tukaani - org.apache.pulsar.functions.runtime.shaded.org.tukaani - - - com.github - org.apache.pulsar.functions.runtime.shaded.com.github - - - commons-io - org.apache.pulsar.functions.runtime.shaded.commons-io - - - org.apache.distributedlog - org.apache.pulsar.functions.runtime.shaded.org.apache.distributedlog - - - - com.fasterxml - org.apache.pulsar.functions.runtime.shaded.com.fasterxml - - - org.inferred - org.apache.pulsar.functions.runtime.shaded.org.inferred - - - org.apache.bookkeeper - org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper - - - org.bookkeeper - org.apache.pulsar.functions.runtime.shaded.org.bookkeeper - - - dlshade - org.apache.pulsar.functions.runtime.shaded.dlshade - - - - org.codehaus.jackson - org.apache.pulsar.functions.runtime.shaded.org.codehaus.jackson - - - net.java.dev.jna - org.apache.pulsar.functions.runtime.shaded.net.java.dev.jna - - - org.apache.curator - org.apache.pulsar.functions.runtime.shaded.org.apache.curator - - - javax.validation - org.apache.pulsar.functions.runtime.shaded.javax.validation - - - javax.activation - org.apache.pulsar.functions.runtime.shaded.javax.activation - - - io.prometheus - org.apache.pulsar.functions.runtime.shaded.io.prometheus - - - io.prometheus.jmx - org.apache.pulsar.functions.runtime.shaded.io.prometheus.jmx - - - org.apache.zookeeper - org.apache.pulsar.functions.runtime.shaded.org.apache.zookeeper - - - io.jsonwebtoken - org.apache.pulsar.functions.runtime.shaded.io.jsonwebtoken - - - commons-codec - org.apache.pulsar.functions.runtime.shaded.commons-codec - - - com.thoughtworks.paranamer - org.apache.pulsar.functions.runtime.shaded.com.thoughtworks.paranamer - - - org.codehaus.mojo - org.apache.pulsar.functions.runtime.shaded.org.codehaus.mojo - - - com.github.luben - org.apache.pulsar.functions.runtime.shaded.com.github.luben - - - jline - org.apache.pulsar.functions.runtime.shaded.jline - - - org.xerial.snappy - org.apache.pulsar.functions.runtime.shaded.org.xerial.snappy - - - javax.annotation - org.apache.pulsar.functions.runtime.shaded.javax.annotation - - - org.checkerframework - org.apache.pulsar.functions.runtime.shaded.org.checkerframework - - - org.apache.yetus - org.apache.pulsar.functions.runtime.shaded.org.apache.yetus - - - commons-cli - org.apache.pulsar.functions.runtime.shaded.commons-cli - - - commons-lang - org.apache.pulsar.functions.runtime.shaded.commons-lang - - - com.squareup.okio - org.apache.pulsar.functions.runtime.shaded.com.squareup.okio - - - org.rocksdb - org.apache.pulsar.functions.runtime.shaded.org.rocksdb - - - org.objenesis - org.apache.pulsar.functions.runtime.shaded.org.objenesis - - - org.eclipse.jetty - org.apache.pulsar.functions.runtime.shaded.org.eclipse.jetty - - - org.apache.avro - org.apache.pulsar.functions.runtime.shaded.org.apache.avro - - - avro.shaded - org.apache.pulsar.functions.runtime.shaded.avo.shaded - - - com.yahoo.datasketches - org.apache.pulsar.shaded.com.yahoo.datasketches - - - com.yahoo.sketches - org.apache.pulsar.shaded.com.yahoo.sketches - - - com.beust - org.apache.pulsar.functions.runtime.shaded.com.beust - - - - io.netty - org.apache.pulsar.functions.runtime.shaded.io.netty - - - org.hamcrest - org.apache.pulsar.functions.runtime.shaded.org.hamcrest - - - aj.org - org.apache.pulsar.functions.runtime.shaded.aj.org - - - com.scurrilous - org.apache.pulsar.functions.runtime.shaded.com.scurrilous - - - okio - org.apache.pulsar.functions.runtime.shaded.okio - - + + com.fasterxml + org.apache.pulsar.functions.runtime.shaded.com.fasterxml + + + org.inferred + org.apache.pulsar.functions.runtime.shaded.org.inferred + + + org.apache.bookkeeper + org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper + + + org.bookkeeper + org.apache.pulsar.functions.runtime.shaded.org.bookkeeper + + + dlshade + org.apache.pulsar.functions.runtime.shaded.dlshade + + + + org.codehaus.jackson + org.apache.pulsar.functions.runtime.shaded.org.codehaus.jackson + + + net.java.dev.jna + org.apache.pulsar.functions.runtime.shaded.net.java.dev.jna + + + org.apache.curator + org.apache.pulsar.functions.runtime.shaded.org.apache.curator + + + javax.validation + org.apache.pulsar.functions.runtime.shaded.javax.validation + + + javax.activation + org.apache.pulsar.functions.runtime.shaded.javax.activation + + + io.prometheus + org.apache.pulsar.functions.runtime.shaded.io.prometheus + + + io.prometheus.jmx + org.apache.pulsar.functions.runtime.shaded.io.prometheus.jmx + + + org.apache.zookeeper + org.apache.pulsar.functions.runtime.shaded.org.apache.zookeeper + + + io.jsonwebtoken + org.apache.pulsar.functions.runtime.shaded.io.jsonwebtoken + + + commons-codec + org.apache.pulsar.functions.runtime.shaded.commons-codec + + + com.thoughtworks.paranamer + org.apache.pulsar.functions.runtime.shaded.com.thoughtworks.paranamer + + + org.codehaus.mojo + org.apache.pulsar.functions.runtime.shaded.org.codehaus.mojo + + + com.github.luben + org.apache.pulsar.functions.runtime.shaded.com.github.luben + + + jline + org.apache.pulsar.functions.runtime.shaded.jline + + + org.xerial.snappy + org.apache.pulsar.functions.runtime.shaded.org.xerial.snappy + + + javax.annotation + org.apache.pulsar.functions.runtime.shaded.javax.annotation + + + org.checkerframework + org.apache.pulsar.functions.runtime.shaded.org.checkerframework + + + org.apache.yetus + org.apache.pulsar.functions.runtime.shaded.org.apache.yetus + + + commons-cli + org.apache.pulsar.functions.runtime.shaded.commons-cli + + + commons-lang + org.apache.pulsar.functions.runtime.shaded.commons-lang + + + com.squareup.okio + org.apache.pulsar.functions.runtime.shaded.com.squareup.okio + + + org.rocksdb + org.apache.pulsar.functions.runtime.shaded.org.rocksdb + + + org.objenesis + org.apache.pulsar.functions.runtime.shaded.org.objenesis + + + org.eclipse.jetty + org.apache.pulsar.functions.runtime.shaded.org.eclipse.jetty + + + org.apache.avro + org.apache.pulsar.functions.runtime.shaded.org.apache.avro + + + avro.shaded + org.apache.pulsar.functions.runtime.shaded.avo.shaded + + + com.yahoo.datasketches + org.apache.pulsar.shaded.com.yahoo.datasketches + + + com.yahoo.sketches + org.apache.pulsar.shaded.com.yahoo.sketches + + + com.beust + org.apache.pulsar.functions.runtime.shaded.com.beust + + + + io.netty + org.apache.pulsar.functions.runtime.shaded.io.netty + + + org.hamcrest + org.apache.pulsar.functions.runtime.shaded.org.hamcrest + + + aj.org + org.apache.pulsar.functions.runtime.shaded.aj.org + + + com.scurrilous + org.apache.pulsar.functions.runtime.shaded.com.scurrilous + + + okio + org.apache.pulsar.functions.runtime.shaded.okio + + - - org.asynchttpclient - org.apache.pulsar.functions.runtime.shaded.org.asynchttpclient - - - io.airlift - org.apache.pulsar.functions.runtime.shaded.io.airlift - - - - - - - - - + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + + + diff --git a/pulsar-functions/localrun/pom.xml b/pulsar-functions/localrun/pom.xml index e9c020b52b545..712e98ea0f24b 100644 --- a/pulsar-functions/localrun/pom.xml +++ b/pulsar-functions/localrun/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar - pulsar-functions - 3.2.0-SNAPSHOT - .. - - - pulsar-functions-local-runner-original - Pulsar Functions :: Local Runner original - - - - ${project.groupId} - pulsar-functions-runtime - ${project.parent.version} - - - - - org.apache.logging.log4j - log4j-slf4j-impl - - - org.apache.logging.log4j - log4j-api - - - org.apache.logging.log4j - log4j-core - - - - io.grpc - grpc-all - - - - io.perfmark - perfmark-api - runtime - - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - check - verify - - check - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - + + 4.0.0 + + io.streamnative + pulsar-functions + 3.2.0-SNAPSHOT + .. + + pulsar-functions-local-runner-original + Pulsar Functions :: Local Runner original + + + ${project.groupId} + pulsar-functions-runtime + ${project.parent.version} + + + + org.apache.logging.log4j + log4j-slf4j-impl + + + org.apache.logging.log4j + log4j-api + + + org.apache.logging.log4j + log4j-core + + + io.grpc + grpc-all + + + io.perfmark + perfmark-api + runtime + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + check + verify + + check + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + diff --git a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java index ed9b0af3b43d8..711fa33edb2a2 100644 --- a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java +++ b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java @@ -52,7 +52,9 @@ import lombok.Value; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.common.functions.FunctionConfig; +import org.apache.pulsar.common.functions.FunctionDefinition; import org.apache.pulsar.common.functions.Utils; +import org.apache.pulsar.common.io.ConnectorDefinition; import org.apache.pulsar.common.io.SinkConfig; import org.apache.pulsar.common.io.SourceConfig; import org.apache.pulsar.common.nar.FileUtils; @@ -75,8 +77,11 @@ import org.apache.pulsar.functions.secretsproviderconfigurator.SecretsProviderConfigurator; import org.apache.pulsar.functions.utils.FunctionCommon; import org.apache.pulsar.functions.utils.FunctionConfigUtils; +import org.apache.pulsar.functions.utils.FunctionRuntimeCommon; +import org.apache.pulsar.functions.utils.LoadedFunctionPackage; import org.apache.pulsar.functions.utils.SinkConfigUtils; import org.apache.pulsar.functions.utils.SourceConfigUtils; +import org.apache.pulsar.functions.utils.ValidatableFunctionPackage; import org.apache.pulsar.functions.utils.functioncache.FunctionCacheEntry; import org.apache.pulsar.functions.utils.functions.FunctionArchive; import org.apache.pulsar.functions.utils.functions.FunctionUtils; @@ -357,9 +362,12 @@ public void start(boolean blocking) throws Exception { userCodeFile = functionConfig.getJar(); userCodeClassLoader = extractClassLoader( userCodeFile, ComponentType.FUNCTION, functionConfig.getClassName()); + ValidatableFunctionPackage validatableFunctionPackage = + new LoadedFunctionPackage(getCurrentOrUserCodeClassLoader(), + FunctionDefinition.class); functionDetails = FunctionConfigUtils.convert( functionConfig, - FunctionConfigUtils.validateJavaFunction(functionConfig, getCurrentOrUserCodeClassLoader())); + FunctionConfigUtils.validateJavaFunction(functionConfig, validatableFunctionPackage)); } else if (functionConfig.getRuntime() == FunctionConfig.Runtime.GO) { userCodeFile = functionConfig.getGo(); } else if (functionConfig.getRuntime() == FunctionConfig.Runtime.PYTHON) { @@ -369,7 +377,10 @@ public void start(boolean blocking) throws Exception { } if (functionDetails == null) { - functionDetails = FunctionConfigUtils.convert(functionConfig, getCurrentOrUserCodeClassLoader()); + ValidatableFunctionPackage validatableFunctionPackage = + new LoadedFunctionPackage(getCurrentOrUserCodeClassLoader(), + FunctionDefinition.class); + functionDetails = FunctionConfigUtils.convert(functionConfig, validatableFunctionPackage); } } else if (sourceConfig != null) { inferMissingArguments(sourceConfig); @@ -377,9 +388,10 @@ public void start(boolean blocking) throws Exception { parallelism = sourceConfig.getParallelism(); userCodeClassLoader = extractClassLoader( userCodeFile, ComponentType.SOURCE, sourceConfig.getClassName()); - functionDetails = SourceConfigUtils.convert( - sourceConfig, - SourceConfigUtils.validateAndExtractDetails(sourceConfig, getCurrentOrUserCodeClassLoader(), true)); + ValidatableFunctionPackage validatableFunctionPackage = + new LoadedFunctionPackage(getCurrentOrUserCodeClassLoader(), ConnectorDefinition.class); + functionDetails = SourceConfigUtils.convert(sourceConfig, + SourceConfigUtils.validateAndExtractDetails(sourceConfig, validatableFunctionPackage, true)); } else if (sinkConfig != null) { inferMissingArguments(sinkConfig); userCodeFile = sinkConfig.getArchive(); @@ -387,6 +399,8 @@ public void start(boolean blocking) throws Exception { parallelism = sinkConfig.getParallelism(); userCodeClassLoader = extractClassLoader( userCodeFile, ComponentType.SINK, sinkConfig.getClassName()); + ValidatableFunctionPackage validatableFunctionPackage = + new LoadedFunctionPackage(getCurrentOrUserCodeClassLoader(), ConnectorDefinition.class); if (isNotEmpty(sinkConfig.getTransformFunction())) { transformFunctionCodeClassLoader = extractClassLoader( sinkConfig.getTransformFunction(), @@ -395,16 +409,19 @@ public void start(boolean blocking) throws Exception { } ClassLoader functionClassLoader = null; + ValidatableFunctionPackage validatableTransformFunction = null; if (transformFunctionCodeClassLoader != null) { functionClassLoader = transformFunctionCodeClassLoader.getClassLoader() == null ? Thread.currentThread().getContextClassLoader() : transformFunctionCodeClassLoader.getClassLoader(); + validatableTransformFunction = + new LoadedFunctionPackage(functionClassLoader, FunctionDefinition.class); } functionDetails = SinkConfigUtils.convert( sinkConfig, - SinkConfigUtils.validateAndExtractDetails(sinkConfig, getCurrentOrUserCodeClassLoader(), - functionClassLoader, true)); + SinkConfigUtils.validateAndExtractDetails(sinkConfig, validatableFunctionPackage, + validatableTransformFunction, true)); } else { throw new IllegalArgumentException("Must specify Function, Source or Sink config"); } @@ -472,7 +489,7 @@ private UserCodeClassLoader extractClassLoader(String userCodeFile, ComponentTyp if (classLoader == null) { if (userCodeFile != null && Utils.isFunctionPackageUrlSupported(userCodeFile)) { File file = FunctionCommon.extractFileFromPkgURL(userCodeFile); - classLoader = FunctionCommon.getClassLoaderFromPackage( + classLoader = FunctionRuntimeCommon.getClassLoaderFromPackage( componentType, className, file, narExtractionDirectory); classLoaderCreated = true; } else if (userCodeFile != null) { @@ -494,7 +511,7 @@ private UserCodeClassLoader extractClassLoader(String userCodeFile, ComponentTyp } throw new RuntimeException(errorMsg + " (" + userCodeFile + ") does not exist"); } - classLoader = FunctionCommon.getClassLoaderFromPackage( + classLoader = FunctionRuntimeCommon.getClassLoaderFromPackage( componentType, className, file, narExtractionDirectory); classLoaderCreated = true; } else { @@ -713,7 +730,7 @@ private ClassLoader isBuiltInFunction(String functionType) throws IOException { FunctionArchive function = functions.get(functionName); if (function != null && function.getFunctionDefinition().getFunctionClass() != null) { // Function type is a valid built-in type. - return function.getClassLoader(); + return function.getFunctionPackage().getClassLoader(); } else { return null; } @@ -727,7 +744,7 @@ private ClassLoader isBuiltInSource(String sourceType) throws IOException { Connector connector = connectors.get(source); if (connector != null && connector.getConnectorDefinition().getSourceClass() != null) { // Source type is a valid built-in connector type. - return connector.getClassLoader(); + return connector.getConnectorFunctionPackage().getClassLoader(); } else { return null; } @@ -741,18 +758,18 @@ private ClassLoader isBuiltInSink(String sinkType) throws IOException { Connector connector = connectors.get(sink); if (connector != null && connector.getConnectorDefinition().getSinkClass() != null) { // Sink type is a valid built-in connector type - return connector.getClassLoader(); + return connector.getConnectorFunctionPackage().getClassLoader(); } else { return null; } } private TreeMap getFunctions() throws IOException { - return FunctionUtils.searchForFunctions(functionsDir); + return FunctionUtils.searchForFunctions(functionsDir, narExtractionDirectory, true); } private TreeMap getConnectors() throws IOException { - return ConnectorUtils.searchForConnectors(connectorsDir, narExtractionDirectory); + return ConnectorUtils.searchForConnectors(connectorsDir, narExtractionDirectory, true); } private SecretsProviderConfigurator getSecretsProviderConfigurator() { diff --git a/pulsar-functions/localrun/src/main/resources/findbugsExclude.xml b/pulsar-functions/localrun/src/main/resources/findbugsExclude.xml index ad183eb5d9427..c7c41fe9ea252 100644 --- a/pulsar-functions/localrun/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/localrun/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + + diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index a96965cb08fea..968cedbfd189a 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT - pulsar-functions Pulsar Functions :: Parent - main @@ -41,24 +39,23 @@ !true - - proto - api-java - java-examples - java-examples-builtin - utils - instance - runtime - runtime-all - worker - secrets - localrun - localrun-shaded - + + proto + api-java + java-examples + java-examples-builtin + utils + instance + runtime + runtime-all + worker + secrets + localrun + localrun-shaded + - - - core-modules + + core-modules proto api-java @@ -72,8 +69,6 @@ secrets localrun - - + - diff --git a/pulsar-functions/proto/pom.xml b/pulsar-functions/proto/pom.xml index 306aaa11d7b3f..65a79c2ea50db 100644 --- a/pulsar-functions/proto/pom.xml +++ b/pulsar-functions/proto/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar - pulsar-functions - 3.2.0-SNAPSHOT - - - pulsar-functions-proto - Pulsar Functions :: Proto - - - - - com.google.protobuf - protobuf-java - - - - com.google.protobuf - protobuf-java-util - - - - javax.annotation - javax.annotation-api - - - - io.grpc - grpc-all - - - io.netty - * - - - - - - io.perfmark - perfmark-api - runtime - - - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - ${protobuf-maven-plugin.version} - - com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} - true - grpc-java - io.grpc:protoc-gen-grpc-java:${protoc-gen-grpc-java.version}:exe:${os.detected.classifier} - - - - generate-sources - - compile - compile-custom - - - - - - - + + 4.0.0 + + io.streamnative + pulsar-functions + 3.2.0-SNAPSHOT + + pulsar-functions-proto + Pulsar Functions :: Proto + + + com.google.protobuf + protobuf-java + + + com.google.protobuf + protobuf-java-util + + + javax.annotation + javax.annotation-api + + + io.grpc + grpc-all + + + io.netty + * + + + + + io.perfmark + perfmark-api + runtime + + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + ${protobuf-maven-plugin.version} + + com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} + true + grpc-java + io.grpc:protoc-gen-grpc-java:${protoc-gen-grpc-java.version}:exe:${os.detected.classifier} + + + + generate-sources + + compile + compile-custom + + + + + + diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml index b9d70b021082c..98be98a41c8f5 100644 --- a/pulsar-functions/runtime-all/pom.xml +++ b/pulsar-functions/runtime-all/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-functions 3.2.0-SNAPSHOT .. - - pulsar-functions-runtime-all Pulsar Functions :: Runtime All - ${project.groupId} pulsar-io-core ${project.version} - ${project.groupId} pulsar-functions-api ${project.version} - ${project.groupId} pulsar-client-api ${project.version} - org.apache.avro avro ${avro.version} - com.fasterxml.jackson.core jackson-databind - com.google.protobuf @@ -95,30 +85,24 @@ gson ${gson.version} - - org.slf4j slf4j-api - org.apache.logging.log4j log4j-slf4j-impl - org.apache.logging.log4j log4j-api - org.apache.logging.log4j log4j-core - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + diff --git a/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml b/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml index 190d9be92940b..84e6ce44f51e4 100644 --- a/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml +++ b/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml @@ -1,3 +1,4 @@ + - pulsar-functions-instance - 30 - - - pulsar.log.appender - RollingFile - - - pulsar.log.level - info - - - bk.log.level - info - - - - - Console - SYSTEM_OUT - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - RollingFile - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.log - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz - true - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - 1 - true - - - 1 GB - - - 0 0 0 * * ? - - - - - ${sys:pulsar.function.log.dir} - 2 - - */${sys:pulsar.function.log.file}*log.gz - - - 30d - - - - - - BkRollingFile - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz - true - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - 1 - true - - - 1 GB - - - 0 0 0 * * ? - - - - - ${sys:pulsar.function.log.dir} - 2 - - */${sys:pulsar.function.log.file}.bk*log.gz - - - 30d - - - - - - - - org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper - ${sys:bk.log.level} - false - - BkRollingFile - - - - info - - ${sys:pulsar.log.appender} - ${sys:pulsar.log.level} - - - - \ No newline at end of file + pulsar-functions-instance + 30 + + + pulsar.log.appender + RollingFile + + + pulsar.log.level + info + + + bk.log.level + info + + + + + Console + SYSTEM_OUT + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + RollingFile + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.log + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz + true + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + 1 + true + + + 1 GB + + + 0 0 0 * * ? + + + + + ${sys:pulsar.function.log.dir} + 2 + + */${sys:pulsar.function.log.file}*log.gz + + + 30d + + + + + + BkRollingFile + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz + true + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + 1 + true + + + 1 GB + + + 0 0 0 * * ? + + + + + ${sys:pulsar.function.log.dir} + 2 + + */${sys:pulsar.function.log.file}.bk*log.gz + + + 30d + + + + + + + + org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper + ${sys:bk.log.level} + false + + BkRollingFile + + + + info + + ${sys:pulsar.log.appender} + ${sys:pulsar.log.level} + + + + diff --git a/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml b/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml index f86d03e41793f..fe478be5b8667 100644 --- a/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml +++ b/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml @@ -1,3 +1,4 @@ + - pulsar-functions-kubernetes-instance - 30 - - - pulsar.log.level - info - - - bk.log.level - info - - - - - Console - SYSTEM_OUT - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - - - org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper - ${sys:bk.log.level} - false - - Console - - - - info - - Console - ${sys:pulsar.log.level} - - - - \ No newline at end of file + pulsar-functions-kubernetes-instance + 30 + + + pulsar.log.level + info + + + bk.log.level + info + + + + + Console + SYSTEM_OUT + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + + + org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper + ${sys:bk.log.level} + false + + Console + + + + info + + Console + ${sys:pulsar.log.level} + + + + diff --git a/pulsar-functions/runtime-all/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceDepsTest.java b/pulsar-functions/runtime-all/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceDepsTest.java index 854c146893243..b65bf17f70b44 100644 --- a/pulsar-functions/runtime-all/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceDepsTest.java +++ b/pulsar-functions/runtime-all/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceDepsTest.java @@ -46,6 +46,8 @@ * 8. Apache AVRO * 9. Jackson Mapper and Databind (dependency of AVRO) * 10. Apache Commons Compress (dependency of AVRO) + * 11. Apache Commons Lang (dependency of Apache Commons Compress) + * 12. Apache Commons IO (dependency of Apache Commons Compress) */ public class JavaInstanceDepsTest { @@ -71,6 +73,8 @@ public void testInstanceJarDeps() throws IOException { && !name.startsWith("org/apache/avro") && !name.startsWith("com/fasterxml/jackson") && !name.startsWith("org/apache/commons/compress") + && !name.startsWith("org/apache/commons/lang3") + && !name.startsWith("org/apache/commons/io") && !name.startsWith("com/google") && !name.startsWith("org/checkerframework") && !name.startsWith("javax/annotation") diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml index 72e24319885fd..a917d280cd492 100644 --- a/pulsar-functions/runtime/pom.xml +++ b/pulsar-functions/runtime/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-functions 3.2.0-SNAPSHOT - pulsar-functions-runtime Pulsar Functions :: Runtime - - ${project.groupId} pulsar-functions-instance ${project.version} - ${project.groupId} pulsar-functions-secrets ${project.version} - com.beust jcommander - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.fasterxml.jackson.core jackson-databind - io.kubernetes client-java @@ -84,9 +75,7 @@ pulsar-broker-common ${project.version} - - @@ -106,7 +95,6 @@ - org.apache.maven.plugins @@ -154,5 +142,4 @@ - diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java index 89281a2f550e2..e23838cb34396 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java @@ -38,6 +38,9 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; +import net.bytebuddy.description.type.TypeDefinition; +import net.bytebuddy.dynamic.ClassFileLocator; +import net.bytebuddy.pool.TypePool; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.common.functions.WindowConfig; import org.apache.pulsar.common.nar.NarClassLoader; @@ -325,7 +328,8 @@ public void close() { } private void inferringMissingTypeClassName(Function.FunctionDetails.Builder functionDetailsBuilder, - ClassLoader classLoader) throws ClassNotFoundException { + ClassLoader classLoader) { + TypePool typePool = TypePool.Default.of(ClassFileLocator.ForClassLoader.of(classLoader)); switch (functionDetailsBuilder.getComponentType()) { case FUNCTION: if ((functionDetailsBuilder.hasSource() @@ -344,14 +348,13 @@ private void inferringMissingTypeClassName(Function.FunctionDetails.Builder func WindowConfig.class); className = windowConfig.getActualWindowFunctionClassName(); } - - Class[] typeArgs = FunctionCommon.getFunctionTypes(classLoader.loadClass(className), + TypeDefinition[] typeArgs = FunctionCommon.getFunctionTypes(typePool.describe(className).resolve(), isWindowConfigPresent); if (functionDetailsBuilder.hasSource() && functionDetailsBuilder.getSource().getTypeClassName().isEmpty() && typeArgs[0] != null) { Function.SourceSpec.Builder sourceBuilder = functionDetailsBuilder.getSource().toBuilder(); - sourceBuilder.setTypeClassName(typeArgs[0].getName()); + sourceBuilder.setTypeClassName(typeArgs[0].asErasure().getTypeName()); functionDetailsBuilder.setSource(sourceBuilder.build()); } @@ -359,7 +362,7 @@ private void inferringMissingTypeClassName(Function.FunctionDetails.Builder func && functionDetailsBuilder.getSink().getTypeClassName().isEmpty() && typeArgs[1] != null) { Function.SinkSpec.Builder sinkBuilder = functionDetailsBuilder.getSink().toBuilder(); - sinkBuilder.setTypeClassName(typeArgs[1].getName()); + sinkBuilder.setTypeClassName(typeArgs[1].asErasure().getTypeName()); functionDetailsBuilder.setSink(sinkBuilder.build()); } } @@ -368,7 +371,8 @@ private void inferringMissingTypeClassName(Function.FunctionDetails.Builder func if ((functionDetailsBuilder.hasSink() && functionDetailsBuilder.getSink().getTypeClassName().isEmpty())) { String typeArg = - getSinkType(functionDetailsBuilder.getSink().getClassName(), classLoader).getName(); + getSinkType(functionDetailsBuilder.getSink().getClassName(), typePool).asErasure() + .getTypeName(); Function.SinkSpec.Builder sinkBuilder = Function.SinkSpec.newBuilder(functionDetailsBuilder.getSink()); @@ -387,7 +391,8 @@ private void inferringMissingTypeClassName(Function.FunctionDetails.Builder func if ((functionDetailsBuilder.hasSource() && functionDetailsBuilder.getSource().getTypeClassName().isEmpty())) { String typeArg = - getSourceType(functionDetailsBuilder.getSource().getClassName(), classLoader).getName(); + getSourceType(functionDetailsBuilder.getSource().getClassName(), typePool).asErasure() + .getTypeName(); Function.SourceSpec.Builder sourceBuilder = Function.SourceSpec.newBuilder(functionDetailsBuilder.getSource()); diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java index ed128568bcf50..9dca4015d5ef5 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java @@ -124,17 +124,17 @@ private static ClassLoader getFunctionClassLoader(InstanceConfig instanceConfig, if (componentType == Function.FunctionDetails.ComponentType.FUNCTION && functionsManager.isPresent()) { return functionsManager.get() .getFunction(instanceConfig.getFunctionDetails().getBuiltin()) - .getClassLoader(); + .getFunctionPackage().getClassLoader(); } if (componentType == Function.FunctionDetails.ComponentType.SOURCE && connectorsManager.isPresent()) { return connectorsManager.get() .getConnector(instanceConfig.getFunctionDetails().getSource().getBuiltin()) - .getClassLoader(); + .getConnectorFunctionPackage().getClassLoader(); } if (componentType == Function.FunctionDetails.ComponentType.SINK && connectorsManager.isPresent()) { return connectorsManager.get() .getConnector(instanceConfig.getFunctionDetails().getSink().getBuiltin()) - .getClassLoader(); + .getConnectorFunctionPackage().getClassLoader(); } } return loadJars(jarFile, instanceConfig, functionId, instanceConfig.getFunctionDetails().getName(), diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/ConnectorsManager.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/ConnectorsManager.java index e1770b8b64415..19d31d0f63b1d 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/ConnectorsManager.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/ConnectorsManager.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.functions.worker; +import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.nio.file.Path; import java.util.List; @@ -27,18 +28,35 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.common.io.ConfigFieldDefinition; import org.apache.pulsar.common.io.ConnectorDefinition; +import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory; import org.apache.pulsar.functions.utils.io.Connector; import org.apache.pulsar.functions.utils.io.ConnectorUtils; @Slf4j -public class ConnectorsManager { +public class ConnectorsManager implements AutoCloseable { @Getter private volatile TreeMap connectors; + @VisibleForTesting + public ConnectorsManager() { + this.connectors = new TreeMap<>(); + } + public ConnectorsManager(WorkerConfig workerConfig) throws IOException { - this.connectors = ConnectorUtils - .searchForConnectors(workerConfig.getConnectorsDirectory(), workerConfig.getNarExtractionDirectory()); + this.connectors = createConnectors(workerConfig); + } + + private static TreeMap createConnectors(WorkerConfig workerConfig) throws IOException { + boolean enableClassloading = workerConfig.getEnableClassloadingOfBuiltinFiles() + || ThreadRuntimeFactory.class.getName().equals(workerConfig.getFunctionRuntimeFactoryClassName()); + return ConnectorUtils.searchForConnectors(workerConfig.getConnectorsDirectory(), + workerConfig.getNarExtractionDirectory(), enableClassloading); + } + + @VisibleForTesting + public void addConnector(String connectorType, Connector connector) { + connectors.put(connectorType, connector); } public Connector getConnector(String connectorType) { @@ -71,7 +89,25 @@ public Path getSinkArchive(String sinkType) { } public void reloadConnectors(WorkerConfig workerConfig) throws IOException { - connectors = ConnectorUtils - .searchForConnectors(workerConfig.getConnectorsDirectory(), workerConfig.getNarExtractionDirectory()); + TreeMap oldConnectors = connectors; + this.connectors = createConnectors(workerConfig); + closeConnectors(oldConnectors); } + + @Override + public void close() { + closeConnectors(connectors); + } + + private void closeConnectors(TreeMap connectorMap) { + connectorMap.values().forEach(connector -> { + try { + connector.close(); + } catch (Exception e) { + log.warn("Failed to close connector", e); + } + }); + connectorMap.clear(); + } + } diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java index 9199d568cad03..5ab7ff7221abb 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.functions.worker; +import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.nio.file.Path; import java.util.List; @@ -25,16 +26,25 @@ import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.common.functions.FunctionDefinition; +import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory; import org.apache.pulsar.functions.utils.functions.FunctionArchive; import org.apache.pulsar.functions.utils.functions.FunctionUtils; @Slf4j -public class FunctionsManager { - +public class FunctionsManager implements AutoCloseable { private TreeMap functions; + @VisibleForTesting + public FunctionsManager() { + this.functions = new TreeMap<>(); + } + public FunctionsManager(WorkerConfig workerConfig) throws IOException { - this.functions = FunctionUtils.searchForFunctions(workerConfig.getFunctionsDirectory()); + this.functions = createFunctions(workerConfig); + } + + public void addFunction(String functionType, FunctionArchive functionArchive) { + functions.put(functionType, functionArchive); } public FunctionArchive getFunction(String functionType) { @@ -51,6 +61,32 @@ public List getFunctionDefinitions() { } public void reloadFunctions(WorkerConfig workerConfig) throws IOException { - this.functions = FunctionUtils.searchForFunctions(workerConfig.getFunctionsDirectory()); + TreeMap oldFunctions = functions; + this.functions = createFunctions(workerConfig); + closeFunctions(oldFunctions); + } + + private static TreeMap createFunctions(WorkerConfig workerConfig) throws IOException { + boolean enableClassloading = workerConfig.getEnableClassloadingOfBuiltinFiles() + || ThreadRuntimeFactory.class.getName().equals(workerConfig.getFunctionRuntimeFactoryClassName()); + return FunctionUtils.searchForFunctions(workerConfig.getFunctionsDirectory(), + workerConfig.getNarExtractionDirectory(), + enableClassloading); + } + + @Override + public void close() { + closeFunctions(functions); + } + + private void closeFunctions(TreeMap functionMap) { + functionMap.values().forEach(functionArchive -> { + try { + functionArchive.close(); + } catch (Exception e) { + log.warn("Failed to close function archive", e); + } + }); + functionMap.clear(); } } diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java index 0ed73953d7aa7..ec0e620d0ae8b 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java @@ -238,6 +238,22 @@ public class WorkerConfig implements Serializable, PulsarConfiguration { ) private boolean zooKeeperAllowReadOnlyOperations; + @FieldContext( + category = CATEGORY_WORKER, + doc = "Specifies if the function worker should use classloading for validating submissions for built-in " + + "connectors and functions. This is required for validateConnectorConfig to take effect. " + + "Default is false." + ) + private Boolean enableClassloadingOfBuiltinFiles = false; + + @FieldContext( + category = CATEGORY_WORKER, + doc = "Specifies if the function worker should use classloading for validating submissions for external " + + "connectors and functions. This is required for validateConnectorConfig to take effect. " + + "Default is false." + ) + private Boolean enableClassloadingOfExternalFiles = false; + @FieldContext( category = CATEGORY_CONNECTORS, doc = "The path to the location to locate builtin connectors" @@ -250,7 +266,22 @@ public class WorkerConfig implements Serializable, PulsarConfiguration { private String narExtractionDirectory = NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR; @FieldContext( category = CATEGORY_CONNECTORS, - doc = "Should we validate connector config during submission" + doc = "Whether to enable referencing connectors directory files by file url in connector (sink/source) " + + "creation. Default is true." + ) + private Boolean enableReferencingConnectorDirectoryFiles = true; + @FieldContext( + category = CATEGORY_FUNCTIONS, + doc = "Regex patterns for enabling creation of connectors by referencing packages in matching http/https " + + "urls." + ) + private List additionalEnabledConnectorUrlPatterns = new ArrayList<>(); + @FieldContext( + category = CATEGORY_CONNECTORS, + doc = "Enables extended validation for connector config with fine-grain annotation based validation " + + "during submission. Classloading with either enableClassloadingOfExternalFiles or " + + "enableClassloadingOfBuiltinFiles must be enabled on the worker for this to take effect. " + + "Default is false." ) private Boolean validateConnectorConfig = false; @FieldContext( @@ -263,6 +294,18 @@ public class WorkerConfig implements Serializable, PulsarConfiguration { doc = "The path to the location to locate builtin functions" ) private String functionsDirectory = "./functions"; + @FieldContext( + category = CATEGORY_FUNCTIONS, + doc = "Whether to enable referencing functions directory files by file url in functions creation. " + + "Default is true." + ) + private Boolean enableReferencingFunctionsDirectoryFiles = true; + @FieldContext( + category = CATEGORY_FUNCTIONS, + doc = "Regex patterns for enabling creation of functions by referencing packages in matching http/https " + + "urls." + ) + private List additionalEnabledFunctionsUrlPatterns = new ArrayList<>(); @FieldContext( category = CATEGORY_FUNC_METADATA_MNG, doc = "The Pulsar topic used for storing function metadata" diff --git a/pulsar-functions/runtime/src/main/resources/findbugsExclude.xml b/pulsar-functions/runtime/src/main/resources/findbugsExclude.xml index b4a4e5926de3f..976abe7c54bb2 100644 --- a/pulsar-functions/runtime/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/runtime/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - - \ No newline at end of file + + + + + + + + + + diff --git a/pulsar-functions/secrets/pom.xml b/pulsar-functions/secrets/pom.xml index 2d664a8374d9e..f6f2107889282 100644 --- a/pulsar-functions/secrets/pom.xml +++ b/pulsar-functions/secrets/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-functions 3.2.0-SNAPSHOT - pulsar-functions-secrets Pulsar Functions :: Secrets - io.kubernetes @@ -50,7 +48,6 @@ - ${project.groupId} pulsar-functions-proto @@ -76,7 +73,6 @@ - com.github.spotbugs spotbugs-maven-plugin @@ -109,5 +105,4 @@ - diff --git a/pulsar-functions/secrets/src/main/resources/findbugsExclude.xml b/pulsar-functions/secrets/src/main/resources/findbugsExclude.xml index 0cab8ae6a418a..7d4464ae1c08d 100644 --- a/pulsar-functions/secrets/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/secrets/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-functions 3.2.0-SNAPSHOT - pulsar-functions-utils Pulsar Functions :: Utils - ${project.groupId} pulsar-common ${project.version} - ${project.groupId} pulsar-io-core ${project.version} - ${project.groupId} pulsar-functions-proto ${project.version} - ${project.groupId} pulsar-functions-api ${project.version} - ${project.groupId} pulsar-config-validation ${project.version} - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.fasterxml.jackson.core jackson-annotations - com.fasterxml.jackson.core jackson-databind - com.google.code.gson gson - net.jodah typetools - - ${project.groupId} - pulsar-client-original - ${project.version} + net.bytebuddy + byte-buddy + + + org.zeroturnaround + zt-zip + 1.17 + + + ${project.groupId} + pulsar-client-original + ${project.version} - ${project.groupId} pulsar-package-core ${project.version} - com.github.tomakehurst wiremock-jre8 ${wiremock.version} test - @@ -126,7 +119,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -142,5 +134,4 @@ - diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java index 7df173da0f195..6a3d2f6ad7ddb 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java @@ -22,16 +22,9 @@ import com.google.protobuf.AbstractMessage.Builder; import com.google.protobuf.MessageOrBuilder; import com.google.protobuf.util.JsonFormat; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.io.ObjectOutputStream; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.ParameterizedType; -import java.lang.reflect.Type; -import java.net.MalformedURLException; import java.net.ServerSocket; import java.net.URISyntaxException; import java.net.URL; @@ -41,10 +34,14 @@ import java.util.Collection; import java.util.Map; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import lombok.AccessLevel; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; -import net.jodah.typetools.TypeResolver; +import net.bytebuddy.description.type.TypeDefinition; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.description.type.TypeList; +import net.bytebuddy.pool.TypePool; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.MessageId; @@ -54,16 +51,11 @@ import org.apache.pulsar.client.impl.auth.AuthenticationDataBasic; import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.functions.Utils; -import org.apache.pulsar.common.nar.NarClassLoader; -import org.apache.pulsar.common.nar.NarClassLoaderBuilder; -import org.apache.pulsar.common.util.ClassLoaderUtils; import org.apache.pulsar.functions.api.Function; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.functions.api.WindowFunction; import org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType; import org.apache.pulsar.functions.proto.Function.FunctionDetails.Runtime; -import org.apache.pulsar.functions.utils.functions.FunctionUtils; -import org.apache.pulsar.functions.utils.io.ConnectorUtils; import org.apache.pulsar.io.core.BatchSource; import org.apache.pulsar.io.core.Sink; import org.apache.pulsar.io.core.Source; @@ -97,50 +89,74 @@ public static int findAvailablePort() { } } - public static Class[] getFunctionTypes(FunctionConfig functionConfig, ClassLoader classLoader) + public static TypeDefinition[] getFunctionTypes(FunctionConfig functionConfig, TypePool typePool) throws ClassNotFoundException { - return getFunctionTypes(functionConfig, classLoader.loadClass(functionConfig.getClassName())); + return getFunctionTypes(functionConfig, typePool.describe(functionConfig.getClassName()).resolve()); } - public static Class[] getFunctionTypes(FunctionConfig functionConfig, Class functionClass) - throws ClassNotFoundException { + public static TypeDefinition[] getFunctionTypes(FunctionConfig functionConfig, TypeDefinition functionClass) { boolean isWindowConfigPresent = functionConfig.getWindowConfig() != null; return getFunctionTypes(functionClass, isWindowConfigPresent); } - public static Class[] getFunctionTypes(Class userClass, boolean isWindowConfigPresent) { + public static TypeDefinition[] getFunctionTypes(TypeDefinition userClass, boolean isWindowConfigPresent) { Class classParent = getFunctionClassParent(userClass, isWindowConfigPresent); - Class[] typeArgs = TypeResolver.resolveRawArguments(classParent, userClass); + TypeList.Generic typeArgsList = resolveInterfaceTypeArguments(userClass, classParent); + TypeDescription.Generic[] typeArgs = new TypeDescription.Generic[2]; + typeArgs[0] = typeArgsList.get(0); + typeArgs[1] = typeArgsList.get(1); // if window function if (isWindowConfigPresent) { if (classParent.equals(java.util.function.Function.class)) { - if (!typeArgs[0].equals(Collection.class)) { + if (!typeArgs[0].asErasure().isAssignableTo(Collection.class)) { throw new IllegalArgumentException("Window function must take a collection as input"); } - typeArgs[0] = (Class) unwrapType(classParent, userClass, 0); + typeArgs[0] = typeArgs[0].getTypeArguments().get(0); } } - if (typeArgs[1].equals(Record.class)) { - typeArgs[1] = (Class) unwrapType(classParent, userClass, 1); + if (typeArgs[1].asErasure().isAssignableTo(Record.class)) { + typeArgs[1] = typeArgs[1].getTypeArguments().get(0); + } + if (typeArgs[1].asErasure().isAssignableTo(CompletableFuture.class)) { + typeArgs[1] = typeArgs[1].getTypeArguments().get(0); } - return typeArgs; } - public static Class[] getRawFunctionTypes(Class userClass, boolean isWindowConfigPresent) { + private static TypeList.Generic resolveInterfaceTypeArguments(TypeDefinition userClass, Class interfaceClass) { + if (!interfaceClass.isInterface()) { + throw new IllegalArgumentException("interfaceClass must be an interface"); + } + for (TypeDescription.Generic interfaze : userClass.getInterfaces()) { + if (interfaze.asErasure().isAssignableTo(interfaceClass)) { + return interfaze.getTypeArguments(); + } + } + if (userClass.getSuperClass() != null) { + return resolveInterfaceTypeArguments(userClass.getSuperClass(), interfaceClass); + } + return null; + } + + public static TypeDescription.Generic[] getRawFunctionTypes(TypeDefinition userClass, + boolean isWindowConfigPresent) { Class classParent = getFunctionClassParent(userClass, isWindowConfigPresent); - return TypeResolver.resolveRawArguments(classParent, userClass); + TypeList.Generic typeArgsList = resolveInterfaceTypeArguments(userClass, classParent); + TypeDescription.Generic[] typeArgs = new TypeDescription.Generic[2]; + typeArgs[0] = typeArgsList.get(0); + typeArgs[1] = typeArgsList.get(1); + return typeArgs; } - public static Class getFunctionClassParent(Class userClass, boolean isWindowConfigPresent) { + public static Class getFunctionClassParent(TypeDefinition userClass, boolean isWindowConfigPresent) { if (isWindowConfigPresent) { - if (WindowFunction.class.isAssignableFrom(userClass)) { + if (userClass.asErasure().isAssignableTo(WindowFunction.class)) { return WindowFunction.class; } else { return java.util.function.Function.class; } } else { - if (Function.class.isAssignableFrom(userClass)) { + if (userClass.asErasure().isAssignableTo(Function.class)) { return Function.class; } else { return java.util.function.Function.class; @@ -148,41 +164,6 @@ public static Class getFunctionClassParent(Class userClass, boolean isWind } } - private static Type unwrapType(Class type, Class subType, int position) { - Type genericType = TypeResolver.resolveGenericType(type, subType); - Type argType = ((ParameterizedType) genericType).getActualTypeArguments()[position]; - return ((ParameterizedType) argType).getActualTypeArguments()[0]; - } - - public static Object createInstance(String userClassName, ClassLoader classLoader) { - Class theCls; - try { - theCls = Class.forName(userClassName); - } catch (ClassNotFoundException | NoClassDefFoundError cnfe) { - try { - theCls = Class.forName(userClassName, true, classLoader); - } catch (ClassNotFoundException | NoClassDefFoundError e) { - throw new RuntimeException("User class must be in class path", cnfe); - } - } - Object result; - try { - Constructor meth = theCls.getDeclaredConstructor(); - meth.setAccessible(true); - result = meth.newInstance(); - } catch (InstantiationException ie) { - throw new RuntimeException("User class must be concrete", ie); - } catch (NoSuchMethodException e) { - throw new RuntimeException("User class doesn't have such method", e); - } catch (IllegalAccessException e) { - throw new RuntimeException("User class must have a no-arg constructor", e); - } catch (InvocationTargetException e) { - throw new RuntimeException("User class constructor throws exception", e); - } - return result; - - } - public static Runtime convertRuntime(FunctionConfig.Runtime runtime) { for (Runtime type : Runtime.values()) { if (type.name().equals(runtime.name())) { @@ -223,29 +204,34 @@ public static FunctionConfig.ProcessingGuarantees convertProcessingGuarantee( throw new RuntimeException("Unrecognized processing guarantee: " + processingGuarantees.name()); } - public static Class getSourceType(String className, ClassLoader classLoader) throws ClassNotFoundException { - return getSourceType(classLoader.loadClass(className)); + public static TypeDefinition getSourceType(String className, TypePool typePool) { + return getSourceType(typePool.describe(className).resolve()); } - public static Class getSourceType(Class sourceClass) { - - if (Source.class.isAssignableFrom(sourceClass)) { - return TypeResolver.resolveRawArgument(Source.class, sourceClass); - } else if (BatchSource.class.isAssignableFrom(sourceClass)) { - return TypeResolver.resolveRawArgument(BatchSource.class, sourceClass); + public static TypeDefinition getSourceType(TypeDefinition sourceClass) { + if (sourceClass.asErasure().isAssignableTo(Source.class)) { + return resolveInterfaceTypeArguments(sourceClass, Source.class).get(0); + } else if (sourceClass.asErasure().isAssignableTo(BatchSource.class)) { + return resolveInterfaceTypeArguments(sourceClass, BatchSource.class).get(0); } else { throw new IllegalArgumentException( String.format("Source class %s does not implement the correct interface", - sourceClass.getName())); + sourceClass.getActualName())); } } - public static Class getSinkType(String className, ClassLoader classLoader) throws ClassNotFoundException { - return getSinkType(classLoader.loadClass(className)); + public static TypeDefinition getSinkType(String className, TypePool typePool) { + return getSinkType(typePool.describe(className).resolve()); } - public static Class getSinkType(Class sinkClass) { - return TypeResolver.resolveRawArgument(Sink.class, sinkClass); + public static TypeDefinition getSinkType(TypeDefinition sinkClass) { + if (sinkClass.asErasure().isAssignableTo(Sink.class)) { + return resolveInterfaceTypeArguments(sinkClass, Sink.class).get(0); + } else { + throw new IllegalArgumentException( + String.format("Sink class %s does not implement the correct interface", + sinkClass.getActualName())); + } } public static void downloadFromHttpUrl(String destPkgUrl, File targetFile) throws IOException { @@ -264,16 +250,6 @@ public static void downloadFromHttpUrl(String destPkgUrl, File targetFile) throw log.info("Downloading function package from {} to {} completed!", destPkgUrl, targetFile.getAbsoluteFile()); } - public static ClassLoader extractClassLoader(String destPkgUrl) throws IOException, URISyntaxException { - File file = extractFileFromPkgURL(destPkgUrl); - try { - return ClassLoaderUtils.loadJar(file); - } catch (MalformedURLException e) { - throw new IllegalArgumentException( - "Corrupt User PackageFile " + file + " with error " + e.getMessage()); - } - } - public static File createPkgTempFile() throws IOException { return File.createTempFile("functions", ".tmp"); } @@ -297,21 +273,6 @@ public static File extractFileFromPkgURL(String destPkgUrl) throws IOException, } } - public static NarClassLoader extractNarClassLoader(File packageFile, - String narExtractionDirectory) { - if (packageFile != null) { - try { - return NarClassLoaderBuilder.builder() - .narFile(packageFile) - .extractionDirectory(narExtractionDirectory) - .build(); - } catch (IOException e) { - throw new IllegalArgumentException(e.getMessage()); - } - } - return null; - } - public static String getFullyQualifiedInstanceId(org.apache.pulsar.functions.proto.Function.Instance instance) { return getFullyQualifiedInstanceId( instance.getFunctionMetaData().getFunctionDetails().getTenant(), @@ -345,17 +306,6 @@ public static final MessageId getMessageId(long sequenceId) { return new MessageIdImpl(ledgerId, entryId, -1); } - public static byte[] toByteArray(Object obj) throws IOException { - byte[] bytes = null; - try (ByteArrayOutputStream bos = new ByteArrayOutputStream(); - ObjectOutputStream oos = new ObjectOutputStream(bos)) { - oos.writeObject(obj); - oos.flush(); - bytes = bos.toByteArray(); - } - return bytes; - } - public static String getUniquePackageName(String packageName) { return String.format("%s-%s", UUID.randomUUID().toString(), packageName); } @@ -403,146 +353,11 @@ private static String extractFromFullyQualifiedName(String fqfn, int index) { throw new RuntimeException("Invalid Fully Qualified Function Name " + fqfn); } - public static Class getTypeArg(String className, Class funClass, ClassLoader classLoader) - throws ClassNotFoundException { - Class loadedClass = classLoader.loadClass(className); - if (!funClass.isAssignableFrom(loadedClass)) { - throw new IllegalArgumentException( - String.format("class %s is not type of %s", className, funClass.getName())); - } - return TypeResolver.resolveRawArgument(funClass, loadedClass); - } - public static double roundDecimal(double value, int places) { double scale = Math.pow(10, places); return Math.round(value * scale) / scale; } - public static ClassLoader getClassLoaderFromPackage( - ComponentType componentType, - String className, - File packageFile, - String narExtractionDirectory) { - String connectorClassName = className; - ClassLoader jarClassLoader = null; - boolean keepJarClassLoader = false; - ClassLoader narClassLoader = null; - boolean keepNarClassLoader = false; - - Exception jarClassLoaderException = null; - Exception narClassLoaderException = null; - - try { - try { - jarClassLoader = ClassLoaderUtils.extractClassLoader(packageFile); - } catch (Exception e) { - jarClassLoaderException = e; - } - try { - narClassLoader = FunctionCommon.extractNarClassLoader(packageFile, narExtractionDirectory); - } catch (Exception e) { - narClassLoaderException = e; - } - - // if connector class name is not provided, we can only try to load archive as a NAR - if (isEmpty(connectorClassName)) { - if (narClassLoader == null) { - throw new IllegalArgumentException(String.format("%s package does not have the correct format. " - + "Pulsar cannot determine if the package is a NAR package or JAR package. " - + "%s classname is not provided and attempts to load it as a NAR package produced " - + "the following error.", - capFirstLetter(componentType), capFirstLetter(componentType)), - narClassLoaderException); - } - try { - if (componentType == ComponentType.FUNCTION) { - connectorClassName = FunctionUtils.getFunctionClass(narClassLoader); - } else if (componentType == ComponentType.SOURCE) { - connectorClassName = ConnectorUtils.getIOSourceClass((NarClassLoader) narClassLoader); - } else { - connectorClassName = ConnectorUtils.getIOSinkClass((NarClassLoader) narClassLoader); - } - } catch (IOException e) { - throw new IllegalArgumentException(String.format("Failed to extract %s class from archive", - componentType.toString().toLowerCase()), e); - } - - try { - narClassLoader.loadClass(connectorClassName); - keepNarClassLoader = true; - return narClassLoader; - } catch (ClassNotFoundException | NoClassDefFoundError e) { - throw new IllegalArgumentException( - String.format("%s class %s must be in class path", capFirstLetter(componentType), - connectorClassName), e); - } - - } else { - // if connector class name is provided, we need to try to load it as a JAR and as a NAR. - if (jarClassLoader != null) { - try { - jarClassLoader.loadClass(connectorClassName); - keepJarClassLoader = true; - return jarClassLoader; - } catch (ClassNotFoundException | NoClassDefFoundError e) { - // class not found in JAR try loading as a NAR and searching for the class - if (narClassLoader != null) { - - try { - narClassLoader.loadClass(connectorClassName); - keepNarClassLoader = true; - return narClassLoader; - } catch (ClassNotFoundException | NoClassDefFoundError e1) { - throw new IllegalArgumentException( - String.format("%s class %s must be in class path", - capFirstLetter(componentType), connectorClassName), e1); - } - } else { - throw new IllegalArgumentException( - String.format("%s class %s must be in class path", capFirstLetter(componentType), - connectorClassName), e); - } - } - } else if (narClassLoader != null) { - try { - narClassLoader.loadClass(connectorClassName); - keepNarClassLoader = true; - return narClassLoader; - } catch (ClassNotFoundException | NoClassDefFoundError e1) { - throw new IllegalArgumentException( - String.format("%s class %s must be in class path", - capFirstLetter(componentType), connectorClassName), e1); - } - } else { - StringBuilder errorMsg = new StringBuilder(capFirstLetter(componentType) - + " package does not have the correct format." - + " Pulsar cannot determine if the package is a NAR package or JAR package."); - - if (jarClassLoaderException != null) { - errorMsg.append( - " Attempts to load it as a JAR package produced error: " + jarClassLoaderException - .getMessage()); - } - - if (narClassLoaderException != null) { - errorMsg.append( - " Attempts to load it as a NAR package produced error: " + narClassLoaderException - .getMessage()); - } - - throw new IllegalArgumentException(errorMsg.toString()); - } - } - } finally { - if (!keepJarClassLoader) { - ClassLoaderUtils.closeClassLoader(jarClassLoader); - } - if (!keepNarClassLoader) { - ClassLoaderUtils.closeClassLoader(narClassLoader); - } - } - } - public static String capFirstLetter(Enum en) { return StringUtils.capitalize(en.toString().toLowerCase()); } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java index e4609672a3d0d..ee59317daf755 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java @@ -18,12 +18,11 @@ */ package org.apache.pulsar.functions.utils; -import static org.apache.commons.lang.StringUtils.isBlank; -import static org.apache.commons.lang.StringUtils.isNotBlank; -import static org.apache.commons.lang.StringUtils.isNotEmpty; +import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isEmpty; +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.commons.lang3.StringUtils.isNotEmpty; import static org.apache.pulsar.common.functions.Utils.BUILTIN; -import static org.apache.pulsar.common.util.ClassLoaderUtils.loadJar; import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromCompressionType; import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromFunctionDetailsCompressionType; import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromFunctionDetailsSubscriptionPosition; @@ -32,9 +31,7 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import java.io.File; -import java.io.IOException; import java.lang.reflect.Type; -import java.net.MalformedURLException; import java.util.Collection; import java.util.HashMap; import java.util.LinkedList; @@ -44,10 +41,13 @@ import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang.StringUtils; +import net.bytebuddy.description.type.TypeDefinition; +import net.bytebuddy.pool.TypePool; +import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.common.functions.ConsumerConfig; import org.apache.pulsar.common.functions.FunctionConfig; +import org.apache.pulsar.common.functions.FunctionDefinition; import org.apache.pulsar.common.functions.ProducerConfig; import org.apache.pulsar.common.functions.Resources; import org.apache.pulsar.common.functions.WindowConfig; @@ -55,7 +55,6 @@ import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.Function.FunctionDetails; -import org.apache.pulsar.functions.utils.functions.FunctionUtils; @Slf4j public class FunctionConfigUtils { @@ -74,26 +73,21 @@ public static class ExtractedFunctionDetails { private static final ObjectMapper OBJECT_MAPPER = ObjectMapperFactory.create(); - public static FunctionDetails convert(FunctionConfig functionConfig, ClassLoader classLoader) - throws IllegalArgumentException { + public static FunctionDetails convert(FunctionConfig functionConfig) { + return convert(functionConfig, (ValidatableFunctionPackage) null); + } - if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) { - if (classLoader != null) { - try { - Class[] typeArgs = FunctionCommon.getFunctionTypes(functionConfig, classLoader); - return convert( - functionConfig, - new ExtractedFunctionDetails( - functionConfig.getClassName(), - typeArgs[0].getName(), - typeArgs[1].getName())); - } catch (ClassNotFoundException | NoClassDefFoundError e) { - throw new IllegalArgumentException( - String.format("Function class %s must be in class path", functionConfig.getClassName()), e); - } - } + public static FunctionDetails convert(FunctionConfig functionConfig, + ValidatableFunctionPackage validatableFunctionPackage) + throws IllegalArgumentException { + if (functionConfig == null) { + throw new IllegalArgumentException("Function config is not provided"); + } + if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA && validatableFunctionPackage != null) { + return convert(functionConfig, doJavaChecks(functionConfig, validatableFunctionPackage)); + } else { + return convert(functionConfig, new ExtractedFunctionDetails(functionConfig.getClassName(), null, null)); } - return convert(functionConfig, new ExtractedFunctionDetails(functionConfig.getClassName(), null, null)); } public static FunctionDetails convert(FunctionConfig functionConfig, ExtractedFunctionDetails extractedDetails) @@ -593,48 +587,49 @@ public static void inferMissingArguments(FunctionConfig functionConfig, } } - public static ExtractedFunctionDetails doJavaChecks(FunctionConfig functionConfig, ClassLoader clsLoader) { + public static ExtractedFunctionDetails doJavaChecks(FunctionConfig functionConfig, + ValidatableFunctionPackage validatableFunctionPackage) { - String functionClassName = functionConfig.getClassName(); - Class functionClass; + String functionClassName = StringUtils.trimToNull(functionConfig.getClassName()); + TypeDefinition functionClass; try { // if class name in function config is not set, this should be a built-in function // thus we should try to find its class name in the NAR service definition if (functionClassName == null) { - try { - functionClassName = FunctionUtils.getFunctionClass(clsLoader); - } catch (IOException e) { - throw new IllegalArgumentException("Failed to extract source class from archive", e); + FunctionDefinition functionDefinition = + validatableFunctionPackage.getFunctionMetaData(FunctionDefinition.class); + if (functionDefinition == null) { + throw new IllegalArgumentException("Function class name is not provided."); + } + functionClassName = functionDefinition.getFunctionClass(); + if (functionClassName == null) { + throw new IllegalArgumentException("Function class name is not provided."); } } - functionClass = clsLoader.loadClass(functionClassName); + functionClass = validatableFunctionPackage.resolveType(functionClassName); - if (!org.apache.pulsar.functions.api.Function.class.isAssignableFrom(functionClass) - && !java.util.function.Function.class.isAssignableFrom(functionClass) - && !org.apache.pulsar.functions.api.WindowFunction.class.isAssignableFrom(functionClass)) { + if (!functionClass.asErasure().isAssignableTo(org.apache.pulsar.functions.api.Function.class) + && !functionClass.asErasure().isAssignableTo(java.util.function.Function.class) + && !functionClass.asErasure() + .isAssignableTo(org.apache.pulsar.functions.api.WindowFunction.class)) { throw new IllegalArgumentException( String.format("Function class %s does not implement the correct interface", - functionClass.getName())); + functionClassName)); } - } catch (ClassNotFoundException | NoClassDefFoundError e) { + } catch (TypePool.Resolution.NoSuchTypeException e) { throw new IllegalArgumentException( - String.format("Function class %s must be in class path", functionConfig.getClassName()), e); + String.format("Function class %s must be in class path", functionClassName), e); } - Class[] typeArgs; - try { - typeArgs = FunctionCommon.getFunctionTypes(functionConfig, functionClass); - } catch (ClassNotFoundException | NoClassDefFoundError e) { - throw new IllegalArgumentException( - String.format("Function class %s must be in class path", functionConfig.getClassName()), e); - } + TypeDefinition[] typeArgs = FunctionCommon.getFunctionTypes(functionConfig, functionClass); // inputs use default schema, so there is no check needed there // Check if the Input serialization/deserialization class exists in jar or already loaded and that it // implements SerDe class if (functionConfig.getCustomSerdeInputs() != null) { functionConfig.getCustomSerdeInputs().forEach((topicName, inputSerializer) -> { - ValidatorUtils.validateSerde(inputSerializer, typeArgs[0], clsLoader, true); + ValidatorUtils.validateSerde(inputSerializer, typeArgs[0], validatableFunctionPackage.getTypePool(), + true); }); } @@ -649,8 +644,8 @@ public static ExtractedFunctionDetails doJavaChecks(FunctionConfig functionConfi throw new IllegalArgumentException( String.format("Topic %s has an incorrect schema Info", topicName)); } - ValidatorUtils.validateSchema(consumerConfig.getSchemaType(), typeArgs[0], clsLoader, true); - + ValidatorUtils.validateSchema(consumerConfig.getSchemaType(), typeArgs[0], + validatableFunctionPackage.getTypePool(), true); }); } @@ -665,13 +660,16 @@ public static ExtractedFunctionDetails doJavaChecks(FunctionConfig functionConfi "Only one of schemaType or serdeClassName should be set in inputSpec"); } if (!isEmpty(conf.getSerdeClassName())) { - ValidatorUtils.validateSerde(conf.getSerdeClassName(), typeArgs[0], clsLoader, true); + ValidatorUtils.validateSerde(conf.getSerdeClassName(), typeArgs[0], + validatableFunctionPackage.getTypePool(), true); } if (!isEmpty(conf.getSchemaType())) { - ValidatorUtils.validateSchema(conf.getSchemaType(), typeArgs[0], clsLoader, true); + ValidatorUtils.validateSchema(conf.getSchemaType(), typeArgs[0], + validatableFunctionPackage.getTypePool(), true); } if (conf.getCryptoConfig() != null) { - ValidatorUtils.validateCryptoKeyReader(conf.getCryptoConfig(), clsLoader, false); + ValidatorUtils.validateCryptoKeyReader(conf.getCryptoConfig(), + validatableFunctionPackage.getTypePool(), false); } }); } @@ -679,8 +677,8 @@ public static ExtractedFunctionDetails doJavaChecks(FunctionConfig functionConfi if (Void.class.equals(typeArgs[1])) { return new FunctionConfigUtils.ExtractedFunctionDetails( functionClassName, - typeArgs[0].getName(), - typeArgs[1].getName()); + typeArgs[0].asErasure().getTypeName(), + typeArgs[1].asErasure().getTypeName()); } // One and only one of outputSchemaType and outputSerdeClassName should be set @@ -690,22 +688,25 @@ public static ExtractedFunctionDetails doJavaChecks(FunctionConfig functionConfi } if (!isEmpty(functionConfig.getOutputSchemaType())) { - ValidatorUtils.validateSchema(functionConfig.getOutputSchemaType(), typeArgs[1], clsLoader, false); + ValidatorUtils.validateSchema(functionConfig.getOutputSchemaType(), typeArgs[1], + validatableFunctionPackage.getTypePool(), false); } if (!isEmpty(functionConfig.getOutputSerdeClassName())) { - ValidatorUtils.validateSerde(functionConfig.getOutputSerdeClassName(), typeArgs[1], clsLoader, false); + ValidatorUtils.validateSerde(functionConfig.getOutputSerdeClassName(), typeArgs[1], + validatableFunctionPackage.getTypePool(), false); } if (functionConfig.getProducerConfig() != null && functionConfig.getProducerConfig().getCryptoConfig() != null) { ValidatorUtils - .validateCryptoKeyReader(functionConfig.getProducerConfig().getCryptoConfig(), clsLoader, true); + .validateCryptoKeyReader(functionConfig.getProducerConfig().getCryptoConfig(), + validatableFunctionPackage.getTypePool(), true); } return new FunctionConfigUtils.ExtractedFunctionDetails( functionClassName, - typeArgs[0].getName(), - typeArgs[1].getName()); + typeArgs[0].asErasure().getTypeName(), + typeArgs[1].asErasure().getTypeName()); } private static void doPythonChecks(FunctionConfig functionConfig) { @@ -912,47 +913,21 @@ public static Collection collectAllInputTopics(FunctionConfig functionCo return retval; } - public static ClassLoader validate(FunctionConfig functionConfig, File functionPackageFile) { + public static void validateNonJavaFunction(FunctionConfig functionConfig) { doCommonChecks(functionConfig); - if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) { - ClassLoader classLoader; - if (functionPackageFile != null) { - try { - classLoader = loadJar(functionPackageFile); - } catch (MalformedURLException e) { - throw new IllegalArgumentException("Corrupted Jar File", e); - } - } else if (!isEmpty(functionConfig.getJar())) { - File jarFile = new File(functionConfig.getJar()); - if (!jarFile.exists()) { - throw new IllegalArgumentException("Jar file does not exist"); - } - try { - classLoader = loadJar(jarFile); - } catch (Exception e) { - throw new IllegalArgumentException("Corrupted Jar File", e); - } - } else { - throw new IllegalArgumentException("Function Package is not provided"); - } - - doJavaChecks(functionConfig, classLoader); - return classLoader; - } else if (functionConfig.getRuntime() == FunctionConfig.Runtime.GO) { + if (functionConfig.getRuntime() == FunctionConfig.Runtime.GO) { doGolangChecks(functionConfig); - return null; } else if (functionConfig.getRuntime() == FunctionConfig.Runtime.PYTHON) { doPythonChecks(functionConfig); - return null; } else { throw new IllegalArgumentException("Function language runtime is either not set or cannot be determined"); } } public static ExtractedFunctionDetails validateJavaFunction(FunctionConfig functionConfig, - ClassLoader classLoader) { + ValidatableFunctionPackage validatableFunctionPackage) { doCommonChecks(functionConfig); - return doJavaChecks(functionConfig, classLoader); + return doJavaChecks(functionConfig, validatableFunctionPackage); } public static FunctionConfig validateUpdate(FunctionConfig existingConfig, FunctionConfig newConfig) { diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionFilePackage.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionFilePackage.java new file mode 100644 index 0000000000000..8224de32521fb --- /dev/null +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionFilePackage.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.utils; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.MalformedURLException; +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.List; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.dynamic.ClassFileLocator; +import net.bytebuddy.pool.TypePool; +import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; +import org.apache.pulsar.functions.utils.functions.FunctionUtils; +import org.zeroturnaround.zip.ZipUtil; + +/** + * FunctionFilePackage is a class that represents a function package and + * implements the ValidatableFunctionPackage interface which decouples the + * function package from classloading. + */ +public class FunctionFilePackage implements AutoCloseable, ValidatableFunctionPackage { + private final File file; + private final ClassFileLocator.Compound classFileLocator; + private final TypePool typePool; + private final boolean isNar; + private final String narExtractionDirectory; + private final boolean enableClassloading; + + private ClassLoader classLoader; + + private final Object configMetadata; + + public FunctionFilePackage(File file, String narExtractionDirectory, boolean enableClassloading, + Class configClass) { + this.file = file; + boolean nonZeroFile = file.isFile() && file.length() > 0; + this.isNar = nonZeroFile ? ZipUtil.containsAnyEntry(file, + new String[] {"META-INF/services/pulsar-io.yaml", "META-INF/bundled-dependencies"}) : false; + this.narExtractionDirectory = narExtractionDirectory; + this.enableClassloading = enableClassloading; + if (isNar) { + List classpathFromArchive = null; + try { + classpathFromArchive = NarClassLoader.getClasspathFromArchive(file, narExtractionDirectory); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + List classFileLocators = new ArrayList<>(); + classFileLocators.add(ClassFileLocator.ForClassLoader.ofSystemLoader()); + for (File classpath : classpathFromArchive) { + if (classpath.exists()) { + try { + ClassFileLocator locator; + if (classpath.isDirectory()) { + locator = new ClassFileLocator.ForFolder(classpath); + } else { + locator = ClassFileLocator.ForJarFile.of(classpath); + } + classFileLocators.add(locator); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + this.classFileLocator = new ClassFileLocator.Compound(classFileLocators); + this.typePool = TypePool.Default.of(classFileLocator); + try { + this.configMetadata = FunctionUtils.getPulsarIOServiceConfig(file, configClass); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } else { + try { + this.classFileLocator = nonZeroFile + ? new ClassFileLocator.Compound(ClassFileLocator.ForClassLoader.ofSystemLoader(), + ClassFileLocator.ForJarFile.of(file)) : + new ClassFileLocator.Compound(ClassFileLocator.ForClassLoader.ofSystemLoader()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + this.typePool = + TypePool.Default.of(classFileLocator); + this.configMetadata = null; + } + } + + public TypeDescription resolveType(String className) { + return typePool.describe(className).resolve(); + } + + public boolean isNar() { + return isNar; + } + + public File getFile() { + return file; + } + + public TypePool getTypePool() { + return typePool; + } + + @Override + public T getFunctionMetaData(Class clazz) { + return configMetadata != null ? clazz.cast(configMetadata) : null; + } + + @Override + public synchronized void close() throws IOException { + classFileLocator.close(); + if (classLoader instanceof Closeable) { + ((Closeable) classLoader).close(); + } + } + + public boolean isEnableClassloading() { + return enableClassloading; + } + + public synchronized ClassLoader getClassLoader() { + if (classLoader == null) { + classLoader = createClassLoader(); + } + return classLoader; + } + + private ClassLoader createClassLoader() { + if (enableClassloading) { + if (isNar) { + try { + return NarClassLoaderBuilder.builder() + .narFile(file) + .extractionDirectory(narExtractionDirectory) + .build(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } else { + try { + return new URLClassLoader(new java.net.URL[] {file.toURI().toURL()}, + NarClassLoader.class.getClassLoader()); + } catch (MalformedURLException e) { + throw new UncheckedIOException(e); + } + } + } else { + throw new IllegalStateException("Classloading is not enabled"); + } + } + + @Override + public String toString() { + return "FunctionFilePackage{" + + "file=" + file + + ", isNar=" + isNar + + '}'; + } +} diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionRuntimeCommon.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionRuntimeCommon.java new file mode 100644 index 0000000000000..ed17478dd00ed --- /dev/null +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionRuntimeCommon.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.utils; + +import static org.apache.commons.lang3.StringUtils.isEmpty; +import java.io.File; +import java.io.IOException; +import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; +import org.apache.pulsar.common.util.ClassLoaderUtils; +import org.apache.pulsar.functions.proto.Function; +import org.apache.pulsar.functions.utils.functions.FunctionUtils; +import org.apache.pulsar.functions.utils.io.ConnectorUtils; + +public class FunctionRuntimeCommon { + public static NarClassLoader extractNarClassLoader(File packageFile, + String narExtractionDirectory) { + if (packageFile != null) { + try { + return NarClassLoaderBuilder.builder() + .narFile(packageFile) + .extractionDirectory(narExtractionDirectory) + .build(); + } catch (IOException e) { + throw new IllegalArgumentException(e.getMessage()); + } + } + return null; + } + + public static ClassLoader getClassLoaderFromPackage( + Function.FunctionDetails.ComponentType componentType, + String className, + File packageFile, + String narExtractionDirectory) { + String connectorClassName = className; + ClassLoader jarClassLoader = null; + boolean keepJarClassLoader = false; + NarClassLoader narClassLoader = null; + boolean keepNarClassLoader = false; + + Exception jarClassLoaderException = null; + Exception narClassLoaderException = null; + + try { + try { + jarClassLoader = ClassLoaderUtils.extractClassLoader(packageFile); + } catch (Exception e) { + jarClassLoaderException = e; + } + try { + narClassLoader = extractNarClassLoader(packageFile, narExtractionDirectory); + } catch (Exception e) { + narClassLoaderException = e; + } + + // if connector class name is not provided, we can only try to load archive as a NAR + if (isEmpty(connectorClassName)) { + if (narClassLoader == null) { + throw new IllegalArgumentException(String.format("%s package does not have the correct format. " + + "Pulsar cannot determine if the package is a NAR package or JAR package. " + + "%s classname is not provided and attempts to load it as a NAR package produced " + + "the following error.", + FunctionCommon.capFirstLetter(componentType), FunctionCommon.capFirstLetter(componentType)), + narClassLoaderException); + } + try { + if (componentType == Function.FunctionDetails.ComponentType.FUNCTION) { + connectorClassName = FunctionUtils.getFunctionClass(narClassLoader); + } else if (componentType == Function.FunctionDetails.ComponentType.SOURCE) { + connectorClassName = ConnectorUtils.getIOSourceClass(narClassLoader); + } else { + connectorClassName = ConnectorUtils.getIOSinkClass(narClassLoader); + } + } catch (IOException e) { + throw new IllegalArgumentException(String.format("Failed to extract %s class from archive", + componentType.toString().toLowerCase()), e); + } + + try { + narClassLoader.loadClass(connectorClassName); + keepNarClassLoader = true; + return narClassLoader; + } catch (ClassNotFoundException | NoClassDefFoundError e) { + throw new IllegalArgumentException(String.format("%s class %s must be in class path", + FunctionCommon.capFirstLetter(componentType), connectorClassName), e); + } + + } else { + // if connector class name is provided, we need to try to load it as a JAR and as a NAR. + if (jarClassLoader != null) { + try { + jarClassLoader.loadClass(connectorClassName); + keepJarClassLoader = true; + return jarClassLoader; + } catch (ClassNotFoundException | NoClassDefFoundError e) { + // class not found in JAR try loading as a NAR and searching for the class + if (narClassLoader != null) { + + try { + narClassLoader.loadClass(connectorClassName); + keepNarClassLoader = true; + return narClassLoader; + } catch (ClassNotFoundException | NoClassDefFoundError e1) { + throw new IllegalArgumentException( + String.format("%s class %s must be in class path", + FunctionCommon.capFirstLetter(componentType), connectorClassName), e1); + } + } else { + throw new IllegalArgumentException(String.format("%s class %s must be in class path", + FunctionCommon.capFirstLetter(componentType), connectorClassName), e); + } + } + } else if (narClassLoader != null) { + try { + narClassLoader.loadClass(connectorClassName); + keepNarClassLoader = true; + return narClassLoader; + } catch (ClassNotFoundException | NoClassDefFoundError e1) { + throw new IllegalArgumentException( + String.format("%s class %s must be in class path", + FunctionCommon.capFirstLetter(componentType), connectorClassName), e1); + } + } else { + StringBuilder errorMsg = new StringBuilder(FunctionCommon.capFirstLetter(componentType) + + " package does not have the correct format." + + " Pulsar cannot determine if the package is a NAR package or JAR package."); + + if (jarClassLoaderException != null) { + errorMsg.append( + " Attempts to load it as a JAR package produced error: " + jarClassLoaderException + .getMessage()); + } + + if (narClassLoaderException != null) { + errorMsg.append( + " Attempts to load it as a NAR package produced error: " + narClassLoaderException + .getMessage()); + } + + throw new IllegalArgumentException(errorMsg.toString()); + } + } + } finally { + if (!keepJarClassLoader) { + ClassLoaderUtils.closeClassLoader(jarClassLoader); + } + if (!keepNarClassLoader) { + ClassLoaderUtils.closeClassLoader(narClassLoader); + } + } + } + +} diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/LoadedFunctionPackage.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/LoadedFunctionPackage.java new file mode 100644 index 0000000000000..e27ed0eca1973 --- /dev/null +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/LoadedFunctionPackage.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.utils; + +import java.io.IOException; +import java.io.UncheckedIOException; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.dynamic.ClassFileLocator; +import net.bytebuddy.pool.TypePool; +import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.functions.utils.functions.FunctionUtils; + +/** + * LoadedFunctionPackage is a class that represents a function package and + * implements the ValidatableFunctionPackage interface which decouples the + * function package from classloading. This implementation is backed by + * a ClassLoader, and it is used when the function package is already loaded + * by a ClassLoader. This is the case in the LocalRunner and in some of + * the unit tests. + */ +public class LoadedFunctionPackage implements ValidatableFunctionPackage { + private final ClassLoader classLoader; + private final Object configMetadata; + private final TypePool typePool; + + public LoadedFunctionPackage(ClassLoader classLoader, Class configMetadataClass, T configMetadata) { + this.classLoader = classLoader; + this.configMetadata = configMetadata; + typePool = TypePool.Default.of( + ClassFileLocator.ForClassLoader.of(classLoader)); + } + + public LoadedFunctionPackage(ClassLoader classLoader, Class configMetadataClass) { + this.classLoader = classLoader; + if (classLoader instanceof NarClassLoader) { + try { + configMetadata = FunctionUtils.getPulsarIOServiceConfig((NarClassLoader) classLoader, + configMetadataClass); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } else { + configMetadata = null; + } + typePool = TypePool.Default.of( + ClassFileLocator.ForClassLoader.of(classLoader)); + } + + @Override + public TypeDescription resolveType(String className) { + return typePool.describe(className).resolve(); + } + + @Override + public TypePool getTypePool() { + return typePool; + } + + @Override + public T getFunctionMetaData(Class clazz) { + return configMetadata != null ? clazz.cast(configMetadata) : null; + } + + @Override + public boolean isEnableClassloading() { + return true; + } + + @Override + public ClassLoader getClassLoader() { + return classLoader; + } +} diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java index 7919d69712600..d93676a106d9a 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java @@ -41,23 +41,23 @@ import lombok.Setter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import net.bytebuddy.description.type.TypeDefinition; +import net.bytebuddy.pool.TypePool; import org.apache.commons.lang.StringUtils; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.common.functions.ConsumerConfig; import org.apache.pulsar.common.functions.FunctionConfig; +import org.apache.pulsar.common.functions.FunctionDefinition; import org.apache.pulsar.common.functions.Resources; import org.apache.pulsar.common.io.ConnectorDefinition; import org.apache.pulsar.common.io.SinkConfig; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.nar.NarClassLoader; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.config.validation.ConfigValidation; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.functions.api.utils.IdentityFunction; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.Function.FunctionDetails; -import org.apache.pulsar.functions.utils.functions.FunctionUtils; -import org.apache.pulsar.functions.utils.io.ConnectorUtils; @Slf4j public class SinkConfigUtils { @@ -402,8 +402,8 @@ public static SinkConfig convertFromDetails(FunctionDetails functionDetails) { } public static ExtractedSinkDetails validateAndExtractDetails(SinkConfig sinkConfig, - ClassLoader sinkClassLoader, - ClassLoader functionClassLoader, + ValidatableFunctionPackage sinkFunction, + ValidatableFunctionPackage transformFunction, boolean validateConnectorConfig) { if (isEmpty(sinkConfig.getTenant())) { throw new IllegalArgumentException("Sink tenant cannot be null"); @@ -443,63 +443,72 @@ public static ExtractedSinkDetails validateAndExtractDetails(SinkConfig sinkConf // if class name in sink config is not set, this should be a built-in sink // thus we should try to find it class name in the NAR service definition if (sinkClassName == null) { - try { - sinkClassName = ConnectorUtils.getIOSinkClass((NarClassLoader) sinkClassLoader); - } catch (IOException e) { - throw new IllegalArgumentException("Failed to extract sink class from archive", e); + ConnectorDefinition connectorDefinition = sinkFunction.getFunctionMetaData(ConnectorDefinition.class); + if (connectorDefinition == null) { + throw new IllegalArgumentException( + "Sink package doesn't contain the META-INF/services/pulsar-io.yaml file."); + } + sinkClassName = connectorDefinition.getSinkClass(); + if (sinkClassName == null) { + throw new IllegalArgumentException("Failed to extract sink class from archive"); } } // check if sink implements the correct interfaces - Class sinkClass; + TypeDefinition sinkClass; try { - sinkClass = sinkClassLoader.loadClass(sinkClassName); - } catch (ClassNotFoundException e) { + sinkClass = sinkFunction.resolveType(sinkClassName); + } catch (TypePool.Resolution.NoSuchTypeException e) { throw new IllegalArgumentException( - String.format("Sink class %s not found in class loader", sinkClassName), e); + String.format("Sink class %s not found", sinkClassName), e); } String functionClassName = sinkConfig.getTransformFunctionClassName(); - Class typeArg; - ClassLoader inputClassLoader; - if (functionClassLoader != null) { + TypeDefinition typeArg; + ValidatableFunctionPackage inputFunction; + if (transformFunction != null) { // if function class name in sink config is not set, this should be a built-in function // thus we should try to find it class name in the NAR service definition if (functionClassName == null) { - try { - functionClassName = FunctionUtils.getFunctionClass(functionClassLoader); - } catch (IOException e) { - throw new IllegalArgumentException("Failed to extract function class from archive", e); + FunctionDefinition functionDefinition = + transformFunction.getFunctionMetaData(FunctionDefinition.class); + if (functionDefinition == null) { + throw new IllegalArgumentException( + "Function package doesn't contain the META-INF/services/pulsar-io.yaml file."); + } + functionClassName = functionDefinition.getFunctionClass(); + if (functionClassName == null) { + throw new IllegalArgumentException("Transform function class name must be set"); } } - Class functionClass; + TypeDefinition functionClass; try { - functionClass = functionClassLoader.loadClass(functionClassName); - } catch (ClassNotFoundException e) { + functionClass = transformFunction.resolveType(functionClassName); + } catch (TypePool.Resolution.NoSuchTypeException e) { throw new IllegalArgumentException( - String.format("Function class %s not found in class loader", functionClassName), e); + String.format("Function class %s not found", functionClassName), e); } // extract type from transform function class - if (!getRawFunctionTypes(functionClass, false)[1].equals(Record.class)) { + if (!getRawFunctionTypes(functionClass, false)[1].asErasure().isAssignableTo(Record.class)) { throw new IllegalArgumentException("Sink transform function output must be of type Record"); } typeArg = getFunctionTypes(functionClass, false)[0]; - inputClassLoader = functionClassLoader; + inputFunction = transformFunction; } else { // extract type from sink class typeArg = getSinkType(sinkClass); - inputClassLoader = sinkClassLoader; + inputFunction = sinkFunction; } if (sinkConfig.getTopicToSerdeClassName() != null) { for (String serdeClassName : sinkConfig.getTopicToSerdeClassName().values()) { - ValidatorUtils.validateSerde(serdeClassName, typeArg, inputClassLoader, true); + ValidatorUtils.validateSerde(serdeClassName, typeArg, inputFunction.getTypePool(), true); } } if (sinkConfig.getTopicToSchemaType() != null) { for (String schemaType : sinkConfig.getTopicToSchemaType().values()) { - ValidatorUtils.validateSchema(schemaType, typeArg, inputClassLoader, true); + ValidatorUtils.validateSchema(schemaType, typeArg, inputFunction.getTypePool(), true); } } @@ -512,23 +521,43 @@ public static ExtractedSinkDetails validateAndExtractDetails(SinkConfig sinkConf throw new IllegalArgumentException("Only one of serdeClassName or schemaType should be set"); } if (!isEmpty(consumerSpec.getSerdeClassName())) { - ValidatorUtils.validateSerde(consumerSpec.getSerdeClassName(), typeArg, inputClassLoader, true); + ValidatorUtils.validateSerde(consumerSpec.getSerdeClassName(), typeArg, + inputFunction.getTypePool(), true); } if (!isEmpty(consumerSpec.getSchemaType())) { - ValidatorUtils.validateSchema(consumerSpec.getSchemaType(), typeArg, inputClassLoader, true); + ValidatorUtils.validateSchema(consumerSpec.getSchemaType(), typeArg, + inputFunction.getTypePool(), true); } if (consumerSpec.getCryptoConfig() != null) { - ValidatorUtils.validateCryptoKeyReader(consumerSpec.getCryptoConfig(), inputClassLoader, false); + ValidatorUtils.validateCryptoKeyReader(consumerSpec.getCryptoConfig(), + inputFunction.getTypePool(), false); } } } - // validate user defined config if enabled and sink is loaded from NAR - if (validateConnectorConfig && sinkClassLoader instanceof NarClassLoader) { - validateSinkConfig(sinkConfig, (NarClassLoader) sinkClassLoader); + if (sinkConfig.getRetainKeyOrdering() != null + && sinkConfig.getRetainKeyOrdering() + && sinkConfig.getProcessingGuarantees() != null + && sinkConfig.getProcessingGuarantees() == FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE) { + throw new IllegalArgumentException( + "When effectively once processing guarantee is specified, retain Key ordering cannot be set"); + } + + if (sinkConfig.getRetainKeyOrdering() != null && sinkConfig.getRetainKeyOrdering() + && sinkConfig.getRetainOrdering() != null && sinkConfig.getRetainOrdering()) { + throw new IllegalArgumentException("Only one of retain ordering or retain key ordering can be set"); } - return new ExtractedSinkDetails(sinkClassName, typeArg.getName(), functionClassName); + // validate user defined config if enabled and classloading is enabled + if (validateConnectorConfig) { + if (sinkFunction.isEnableClassloading()) { + validateSinkConfig(sinkConfig, sinkFunction); + } else { + log.warn("Skipping annotation based validation of sink config as classloading is disabled"); + } + } + + return new ExtractedSinkDetails(sinkClassName, typeArg.asErasure().getTypeName(), functionClassName); } public static Collection collectAllInputTopics(SinkConfig sinkConfig) { @@ -684,29 +713,13 @@ public static SinkConfig validateUpdate(SinkConfig existingConfig, SinkConfig ne return mergedConfig; } - public static void validateSinkConfig(SinkConfig sinkConfig, NarClassLoader narClassLoader) { - - if (sinkConfig.getRetainKeyOrdering() != null - && sinkConfig.getRetainKeyOrdering() - && sinkConfig.getProcessingGuarantees() != null - && sinkConfig.getProcessingGuarantees() == FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE) { - throw new IllegalArgumentException( - "When effectively once processing guarantee is specified, retain Key ordering cannot be set"); - } - - if (sinkConfig.getRetainKeyOrdering() != null && sinkConfig.getRetainKeyOrdering() - && sinkConfig.getRetainOrdering() != null && sinkConfig.getRetainOrdering()) { - throw new IllegalArgumentException("Only one of retain ordering or retain key ordering can be set"); - } - + public static void validateSinkConfig(SinkConfig sinkConfig, ValidatableFunctionPackage sinkFunction) { try { - ConnectorDefinition defn = ConnectorUtils.getConnectorDefinition(narClassLoader); - if (defn.getSinkConfigClass() != null) { - Class configClass = Class.forName(defn.getSinkConfigClass(), true, narClassLoader); + ConnectorDefinition defn = sinkFunction.getFunctionMetaData(ConnectorDefinition.class); + if (defn != null && defn.getSinkConfigClass() != null) { + Class configClass = Class.forName(defn.getSinkConfigClass(), true, sinkFunction.getClassLoader()); validateSinkConfig(sinkConfig, configClass); } - } catch (IOException e) { - throw new IllegalArgumentException("Error validating sink config", e); } catch (ClassNotFoundException e) { throw new IllegalArgumentException("Could not find sink config class", e); } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java index f3be015d73754..a6430bbea4585 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java @@ -35,7 +35,9 @@ import lombok.Setter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import net.jodah.typetools.TypeResolver; +import net.bytebuddy.description.type.TypeDefinition; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.pool.TypePool; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.common.functions.ProducerConfig; import org.apache.pulsar.common.functions.Resources; @@ -44,13 +46,11 @@ import org.apache.pulsar.common.io.SourceConfig; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.nar.NarClassLoader; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.config.validation.ConfigValidation; import org.apache.pulsar.functions.api.utils.IdentityFunction; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.Function.FunctionDetails; -import org.apache.pulsar.functions.utils.io.ConnectorUtils; import org.apache.pulsar.io.core.BatchSource; import org.apache.pulsar.io.core.Source; @@ -294,7 +294,7 @@ public static SourceConfig convertFromDetails(FunctionDetails functionDetails) { } public static ExtractedSourceDetails validateAndExtractDetails(SourceConfig sourceConfig, - ClassLoader sourceClassLoader, + ValidatableFunctionPackage sourceFunction, boolean validateConnectorConfig) { if (isEmpty(sourceConfig.getTenant())) { throw new IllegalArgumentException("Source tenant cannot be null"); @@ -319,29 +319,34 @@ public static ExtractedSourceDetails validateAndExtractDetails(SourceConfig sour // if class name in source config is not set, this should be a built-in source // thus we should try to find it class name in the NAR service definition if (sourceClassName == null) { - try { - sourceClassName = ConnectorUtils.getIOSourceClass((NarClassLoader) sourceClassLoader); - } catch (IOException e) { - throw new IllegalArgumentException("Failed to extract source class from archive", e); + ConnectorDefinition connectorDefinition = sourceFunction.getFunctionMetaData(ConnectorDefinition.class); + if (connectorDefinition == null) { + throw new IllegalArgumentException( + "Source package doesn't contain the META-INF/services/pulsar-io.yaml file."); + } + sourceClassName = connectorDefinition.getSourceClass(); + if (sourceClassName == null) { + throw new IllegalArgumentException("Failed to extract source class from archive"); } } // check if source implements the correct interfaces - Class sourceClass; + TypeDescription sourceClass; try { - sourceClass = sourceClassLoader.loadClass(sourceClassName); - } catch (ClassNotFoundException e) { + sourceClass = sourceFunction.resolveType(sourceClassName); + } catch (TypePool.Resolution.NoSuchTypeException e) { throw new IllegalArgumentException( String.format("Source class %s not found in class loader", sourceClassName), e); } - if (!Source.class.isAssignableFrom(sourceClass) && !BatchSource.class.isAssignableFrom(sourceClass)) { + if (!(sourceClass.asErasure().isAssignableTo(Source.class) || sourceClass.asErasure() + .isAssignableTo(BatchSource.class))) { throw new IllegalArgumentException( - String.format("Source class %s does not implement the correct interface", - sourceClass.getName())); + String.format("Source class %s does not implement the correct interface", + sourceClass.getName())); } - if (BatchSource.class.isAssignableFrom(sourceClass)) { + if (sourceClass.asErasure().isAssignableTo(BatchSource.class)) { if (sourceConfig.getBatchSourceConfig() != null) { validateBatchSourceConfig(sourceConfig.getBatchSourceConfig()); } else { @@ -352,7 +357,14 @@ public static ExtractedSourceDetails validateAndExtractDetails(SourceConfig sour } // extract type from source class - Class typeArg = getSourceType(sourceClass); + TypeDefinition typeArg; + + try { + typeArg = getSourceType(sourceClass); + } catch (Exception e) { + throw new IllegalArgumentException( + String.format("Failed to resolve type for Source class %s", sourceClassName), e); + } // Only one of serdeClassName or schemaType should be set if (!StringUtils.isEmpty(sourceConfig.getSerdeClassName()) && !StringUtils @@ -361,29 +373,30 @@ public static ExtractedSourceDetails validateAndExtractDetails(SourceConfig sour } if (!StringUtils.isEmpty(sourceConfig.getSerdeClassName())) { - ValidatorUtils.validateSerde(sourceConfig.getSerdeClassName(), typeArg, sourceClassLoader, false); + ValidatorUtils.validateSerde(sourceConfig.getSerdeClassName(), typeArg, sourceFunction.getTypePool(), + false); } if (!StringUtils.isEmpty(sourceConfig.getSchemaType())) { - ValidatorUtils.validateSchema(sourceConfig.getSchemaType(), typeArg, sourceClassLoader, false); + ValidatorUtils.validateSchema(sourceConfig.getSchemaType(), typeArg, sourceFunction.getTypePool(), + false); } if (sourceConfig.getProducerConfig() != null && sourceConfig.getProducerConfig().getCryptoConfig() != null) { ValidatorUtils - .validateCryptoKeyReader(sourceConfig.getProducerConfig().getCryptoConfig(), sourceClassLoader, - true); + .validateCryptoKeyReader(sourceConfig.getProducerConfig().getCryptoConfig(), + sourceFunction.getTypePool(), true); } - if (typeArg.equals(TypeResolver.Unknown.class)) { - throw new IllegalArgumentException( - String.format("Failed to resolve type for Source class %s", sourceClassName)); - } - - // validate user defined config if enabled and source is loaded from NAR - if (validateConnectorConfig && sourceClassLoader instanceof NarClassLoader) { - validateSourceConfig(sourceConfig, (NarClassLoader) sourceClassLoader); + // validate user defined config if enabled and classloading is enabled + if (validateConnectorConfig) { + if (sourceFunction.isEnableClassloading()) { + validateSourceConfig(sourceConfig, sourceFunction); + } else { + log.warn("Skipping annotation based validation of sink config as classloading is disabled"); + } } - return new ExtractedSourceDetails(sourceClassName, typeArg.getName()); + return new ExtractedSourceDetails(sourceClassName, typeArg.asErasure().getTypeName()); } @SneakyThrows @@ -524,15 +537,14 @@ public static void validateBatchSourceConfigUpdate(BatchSourceConfig existingCon } } - public static void validateSourceConfig(SourceConfig sourceConfig, NarClassLoader narClassLoader) { + public static void validateSourceConfig(SourceConfig sourceConfig, ValidatableFunctionPackage sourceFunction) { try { - ConnectorDefinition defn = ConnectorUtils.getConnectorDefinition(narClassLoader); - if (defn.getSourceConfigClass() != null) { - Class configClass = Class.forName(defn.getSourceConfigClass(), true, narClassLoader); + ConnectorDefinition defn = sourceFunction.getFunctionMetaData(ConnectorDefinition.class); + if (defn != null && defn.getSourceConfigClass() != null) { + Class configClass = + Class.forName(defn.getSourceConfigClass(), true, sourceFunction.getClassLoader()); validateSourceConfig(sourceConfig, configClass); } - } catch (IOException e) { - throw new IllegalArgumentException("Error validating source config", e); } catch (ClassNotFoundException e) { throw new IllegalArgumentException("Could not find source config class"); } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/ValidatableFunctionPackage.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/ValidatableFunctionPackage.java new file mode 100644 index 0000000000000..8d5aefb6f6785 --- /dev/null +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/ValidatableFunctionPackage.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.utils; + +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.pool.TypePool; + +/** + * This abstraction separates the function and connector definition from classloading, + * enabling validation without the need for classloading. It utilizes Byte Buddy for + * type and annotation resolution. + * + * The function or connector definition is directly extracted from the archive file, + * eliminating the need for classloader initialization. + * + * The getClassLoader method should only be invoked when classloading is enabled. + * Classloading is required in the LocalRunner and in the Functions worker when the + * worker is configured with the 'validateConnectorConfig' set to true. + */ +public interface ValidatableFunctionPackage { + /** + * Resolves the type description for the given class name within the function package. + */ + TypeDescription resolveType(String className); + /** + * Returns the Byte Buddy TypePool instance for the function package. + */ + TypePool getTypePool(); + /** + * Returns the function or connector definition metadata. + * Supports FunctionDefinition and ConnectorDefinition as the metadata type. + */ + T getFunctionMetaData(Class clazz); + /** + * Returns if classloading is enabled for the function package. + */ + boolean isEnableClassloading(); + /** + * Returns the classloader for the function package. The classloader is + * lazily initialized when classloading is enabled. + */ + ClassLoader getClassLoader(); +} diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/ValidatorUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/ValidatorUtils.java index 390671c5606af..8df6a3f261a6e 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/ValidatorUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/ValidatorUtils.java @@ -18,35 +18,40 @@ */ package org.apache.pulsar.functions.utils; -import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isEmpty; -import static org.apache.commons.lang3.StringUtils.isNotBlank; import java.util.Map; import lombok.extern.slf4j.Slf4j; -import net.jodah.typetools.TypeResolver; +import net.bytebuddy.description.type.TypeDefinition; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.pool.TypePool; import org.apache.pulsar.client.api.CryptoKeyReader; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.functions.CryptoConfig; import org.apache.pulsar.common.schema.SchemaType; -import org.apache.pulsar.common.util.ClassLoaderUtils; -import org.apache.pulsar.common.util.Reflections; import org.apache.pulsar.functions.api.SerDe; -import org.apache.pulsar.functions.proto.Function; -import org.apache.pulsar.io.core.Sink; -import org.apache.pulsar.io.core.Source; @Slf4j public class ValidatorUtils { private static final String DEFAULT_SERDE = "org.apache.pulsar.functions.api.utils.DefaultSerDe"; - public static void validateSchema(String schemaType, Class typeArg, ClassLoader clsLoader, + public static void validateSchema(String schemaType, TypeDefinition typeArg, TypePool typePool, boolean input) { if (isEmpty(schemaType) || getBuiltinSchemaType(schemaType) != null) { // If it's empty, we use the default schema and no need to validate // If it's built-in, no need to validate } else { - ClassLoaderUtils.implementsClass(schemaType, Schema.class, clsLoader); - validateSchemaType(schemaType, typeArg, clsLoader, input); + TypeDescription schemaClass = null; + try { + schemaClass = typePool.describe(schemaType).resolve(); + } catch (TypePool.Resolution.NoSuchTypeException e) { + throw new IllegalArgumentException( + String.format("The schema class %s does not exist", schemaType)); + } + if (!schemaClass.asErasure().isAssignableTo(Schema.class)) { + throw new IllegalArgumentException( + String.format("%s does not implement %s", schemaType, Schema.class.getName())); + } + validateSchemaType(schemaClass, typeArg, typePool, input); } } @@ -60,29 +65,32 @@ private static SchemaType getBuiltinSchemaType(String schemaTypeOrClassName) { } - public static void validateCryptoKeyReader(CryptoConfig conf, ClassLoader classLoader, boolean isProducer) { + public static void validateCryptoKeyReader(CryptoConfig conf, TypePool typePool, boolean isProducer) { if (isEmpty(conf.getCryptoKeyReaderClassName())) { return; } - Class cryptoClass; + String cryptoClassName = conf.getCryptoKeyReaderClassName(); + TypeDescription cryptoClass = null; try { - cryptoClass = ClassLoaderUtils.loadClass(conf.getCryptoKeyReaderClassName(), classLoader); - } catch (ClassNotFoundException | NoClassDefFoundError e) { + cryptoClass = typePool.describe(cryptoClassName).resolve(); + } catch (TypePool.Resolution.NoSuchTypeException e) { throw new IllegalArgumentException( - String.format("The crypto key reader class %s does not exist", conf.getCryptoKeyReaderClassName())); + String.format("The crypto key reader class %s does not exist", cryptoClassName)); + } + if (!cryptoClass.asErasure().isAssignableTo(CryptoKeyReader.class)) { + throw new IllegalArgumentException( + String.format("%s does not implement %s", cryptoClassName, CryptoKeyReader.class.getName())); } - ClassLoaderUtils.implementsClass(conf.getCryptoKeyReaderClassName(), CryptoKeyReader.class, classLoader); - try { - cryptoClass.getConstructor(Map.class); - } catch (NoSuchMethodException ex) { + boolean hasConstructor = cryptoClass.getDeclaredMethods().stream() + .anyMatch(method -> method.isConstructor() && method.getParameters().size() == 1 + && method.getParameters().get(0).getType().asErasure().represents(Map.class)); + + if (!hasConstructor) { throw new IllegalArgumentException( String.format("The crypto key reader class %s does not implement the desired constructor.", conf.getCryptoKeyReaderClassName())); - - } catch (SecurityException e) { - throw new IllegalArgumentException("Failed to access crypto key reader class", e); } if (isProducer && (conf.getEncryptionKeys() == null || conf.getEncryptionKeys().length == 0)) { @@ -90,7 +98,7 @@ public static void validateCryptoKeyReader(CryptoConfig conf, ClassLoader classL } } - public static void validateSerde(String inputSerializer, Class typeArg, ClassLoader clsLoader, + public static void validateSerde(String inputSerializer, TypeDefinition typeArg, TypePool typePool, boolean deser) { if (isEmpty(inputSerializer)) { return; @@ -98,154 +106,53 @@ public static void validateSerde(String inputSerializer, Class typeArg, Class if (inputSerializer.equals(DEFAULT_SERDE)) { return; } + TypeDescription serdeClass; try { - Class serdeClass = ClassLoaderUtils.loadClass(inputSerializer, clsLoader); - } catch (ClassNotFoundException | NoClassDefFoundError e) { + serdeClass = typePool.describe(inputSerializer).resolve(); + } catch (TypePool.Resolution.NoSuchTypeException e) { throw new IllegalArgumentException( String.format("The input serialization/deserialization class %s does not exist", inputSerializer)); } - ClassLoaderUtils.implementsClass(inputSerializer, SerDe.class, clsLoader); - - SerDe serDe = (SerDe) Reflections.createInstance(inputSerializer, clsLoader); - if (serDe == null) { - throw new IllegalArgumentException(String.format("The SerDe class %s does not exist", - inputSerializer)); - } - Class[] serDeTypes = TypeResolver.resolveRawArguments(SerDe.class, serDe.getClass()); - - // type inheritance information seems to be lost in generic type - // load the actual type class for verification - Class fnInputClass; - Class serdeInputClass; - try { - fnInputClass = Class.forName(typeArg.getName(), true, clsLoader); - serdeInputClass = Class.forName(serDeTypes[0].getName(), true, clsLoader); - } catch (ClassNotFoundException | NoClassDefFoundError e) { - throw new IllegalArgumentException("Failed to load type class", e); - } + TypeDescription.Generic serDeTypeArg = serdeClass.getInterfaces().stream() + .filter(i -> i.asErasure().isAssignableTo(SerDe.class)) + .findFirst() + .map(i -> i.getTypeArguments().get(0)) + .orElseThrow(() -> new IllegalArgumentException( + String.format("%s does not implement %s", inputSerializer, SerDe.class.getName()))); if (deser) { - if (!fnInputClass.isAssignableFrom(serdeInputClass)) { - throw new IllegalArgumentException("Serializer type mismatch " + typeArg + " vs " + serDeTypes[0]); + if (!serDeTypeArg.asErasure().isAssignableTo(typeArg.asErasure())) { + throw new IllegalArgumentException("Serializer type mismatch " + typeArg.getActualName() + " vs " + + serDeTypeArg.getActualName()); } } else { - if (!serdeInputClass.isAssignableFrom(fnInputClass)) { - throw new IllegalArgumentException("Serializer type mismatch " + typeArg + " vs " + serDeTypes[0]); + if (!serDeTypeArg.asErasure().isAssignableFrom(typeArg.asErasure())) { + throw new IllegalArgumentException("Serializer type mismatch " + typeArg.getActualName() + " vs " + + serDeTypeArg.getActualName()); } } } - private static void validateSchemaType(String schemaClassName, Class typeArg, ClassLoader clsLoader, + private static void validateSchemaType(TypeDefinition schema, TypeDefinition typeArg, TypePool typePool, boolean input) { - Schema schema = (Schema) Reflections.createInstance(schemaClassName, clsLoader); - if (schema == null) { - throw new IllegalArgumentException(String.format("The Schema class %s does not exist", - schemaClassName)); - } - Class[] schemaTypes = TypeResolver.resolveRawArguments(Schema.class, schema.getClass()); - // type inheritance information seems to be lost in generic type - // load the actual type class for verification - Class fnInputClass; - Class schemaInputClass; - try { - fnInputClass = Class.forName(typeArg.getName(), true, clsLoader); - schemaInputClass = Class.forName(schemaTypes[0].getName(), true, clsLoader); - } catch (ClassNotFoundException | NoClassDefFoundError e) { - throw new IllegalArgumentException("Failed to load type class", e); - } + TypeDescription.Generic schemaTypeArg = schema.getInterfaces().stream() + .filter(i -> i.asErasure().isAssignableTo(Schema.class)) + .findFirst() + .map(i -> i.getTypeArguments().get(0)) + .orElse(null); if (input) { - if (!fnInputClass.isAssignableFrom(schemaInputClass)) { + if (!schemaTypeArg.asErasure().isAssignableTo(typeArg.asErasure())) { throw new IllegalArgumentException( - "Schema type mismatch " + typeArg + " vs " + schemaTypes[0]); + "Schema type mismatch " + typeArg.getActualName() + " vs " + schemaTypeArg.getActualName()); } } else { - if (!schemaInputClass.isAssignableFrom(fnInputClass)) { + if (!schemaTypeArg.asErasure().isAssignableFrom(typeArg.asErasure())) { throw new IllegalArgumentException( - "Schema type mismatch " + typeArg + " vs " + schemaTypes[0]); - } - } - } - - - public static void validateFunctionClassTypes(ClassLoader classLoader, - Function.FunctionDetails.Builder functionDetailsBuilder) { - - // validate only if classLoader is provided - if (classLoader == null) { - return; - } - - if (isBlank(functionDetailsBuilder.getClassName())) { - throw new IllegalArgumentException("Function class-name can't be empty"); - } - - // validate function class-type - Class functionClass; - try { - functionClass = classLoader.loadClass(functionDetailsBuilder.getClassName()); - } catch (ClassNotFoundException | NoClassDefFoundError e) { - throw new IllegalArgumentException( - String.format("Function class %s must be in class path", functionDetailsBuilder.getClassName()), e); - } - Class[] typeArgs = FunctionCommon.getFunctionTypes(functionClass, false); - - if (!(org.apache.pulsar.functions.api.Function.class.isAssignableFrom(functionClass)) - && !(java.util.function.Function.class.isAssignableFrom(functionClass))) { - throw new RuntimeException("User class must either be Function or java.util.Function"); - } - - if (functionDetailsBuilder.hasSource() && functionDetailsBuilder.getSource() != null - && isNotBlank(functionDetailsBuilder.getSource().getClassName())) { - try { - String sourceClassName = functionDetailsBuilder.getSource().getClassName(); - String argClassName = FunctionCommon.getTypeArg(sourceClassName, Source.class, classLoader).getName(); - functionDetailsBuilder - .setSource(functionDetailsBuilder.getSourceBuilder().setTypeClassName(argClassName)); - - // if sink-class not present then set same arg as source - if (!functionDetailsBuilder.hasSink() || isBlank(functionDetailsBuilder.getSink().getClassName())) { - functionDetailsBuilder - .setSink(functionDetailsBuilder.getSinkBuilder().setTypeClassName(argClassName)); - } - - } catch (IllegalArgumentException ie) { - throw ie; - } catch (Exception e) { - log.error("Failed to validate source class", e); - throw new IllegalArgumentException("Failed to validate source class-name", e); - } - } else if (isBlank(functionDetailsBuilder.getSourceBuilder().getTypeClassName())) { - // if function-src-class is not present then set function-src type-class according to function class - functionDetailsBuilder - .setSource(functionDetailsBuilder.getSourceBuilder().setTypeClassName(typeArgs[0].getName())); - } - - if (functionDetailsBuilder.hasSink() && functionDetailsBuilder.getSink() != null - && isNotBlank(functionDetailsBuilder.getSink().getClassName())) { - try { - String sinkClassName = functionDetailsBuilder.getSink().getClassName(); - String argClassName = FunctionCommon.getTypeArg(sinkClassName, Sink.class, classLoader).getName(); - functionDetailsBuilder.setSink(functionDetailsBuilder.getSinkBuilder().setTypeClassName(argClassName)); - - // if source-class not present then set same arg as sink - if (!functionDetailsBuilder.hasSource() || isBlank(functionDetailsBuilder.getSource().getClassName())) { - functionDetailsBuilder - .setSource(functionDetailsBuilder.getSourceBuilder().setTypeClassName(argClassName)); - } - - } catch (IllegalArgumentException ie) { - throw ie; - } catch (Exception e) { - log.error("Failed to validate sink class", e); - throw new IllegalArgumentException("Failed to validate sink class-name", e); + "Schema type mismatch " + typeArg.getActualName() + " vs " + schemaTypeArg.getActualName()); } - } else if (isBlank(functionDetailsBuilder.getSinkBuilder().getTypeClassName())) { - // if function-sink-class is not present then set function-sink type-class according to function class - functionDetailsBuilder - .setSink(functionDetailsBuilder.getSinkBuilder().setTypeClassName(typeArgs[1].getName())); } } } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java index 028b57d69c86b..cfb213f34ed72 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java @@ -19,14 +19,50 @@ package org.apache.pulsar.functions.utils.functions; import java.nio.file.Path; -import lombok.Builder; -import lombok.Data; import org.apache.pulsar.common.functions.FunctionDefinition; +import org.apache.pulsar.functions.utils.FunctionFilePackage; +import org.apache.pulsar.functions.utils.ValidatableFunctionPackage; -@Builder -@Data -public class FunctionArchive { - private Path archivePath; - private ClassLoader classLoader; - private FunctionDefinition functionDefinition; +public class FunctionArchive implements AutoCloseable { + private final Path archivePath; + private final FunctionDefinition functionDefinition; + private final String narExtractionDirectory; + private final boolean enableClassloading; + private ValidatableFunctionPackage functionPackage; + private boolean closed; + + public FunctionArchive(Path archivePath, FunctionDefinition functionDefinition, String narExtractionDirectory, + boolean enableClassloading) { + this.archivePath = archivePath; + this.functionDefinition = functionDefinition; + this.narExtractionDirectory = narExtractionDirectory; + this.enableClassloading = enableClassloading; + } + + public Path getArchivePath() { + return archivePath; + } + + public synchronized ValidatableFunctionPackage getFunctionPackage() { + if (closed) { + throw new IllegalStateException("FunctionArchive is already closed"); + } + if (functionPackage == null) { + functionPackage = new FunctionFilePackage(archivePath.toFile(), narExtractionDirectory, enableClassloading, + FunctionDefinition.class); + } + return functionPackage; + } + + public FunctionDefinition getFunctionDefinition() { + return functionDefinition; + } + + @Override + public synchronized void close() throws Exception { + closed = true; + if (functionPackage instanceof AutoCloseable) { + ((AutoCloseable) functionPackage).close(); + } + } } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java index 941df573e495e..31a5540e0bfaf 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.pulsar.functions.utils.functions; import java.io.File; @@ -30,10 +31,8 @@ import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.common.functions.FunctionDefinition; import org.apache.pulsar.common.nar.NarClassLoader; -import org.apache.pulsar.common.nar.NarClassLoaderBuilder; import org.apache.pulsar.common.util.ObjectMapperFactory; -import org.apache.pulsar.functions.api.Function; -import org.apache.pulsar.functions.utils.Exceptions; +import org.zeroturnaround.zip.ZipUtil; @UtilityClass @@ -45,43 +44,40 @@ public class FunctionUtils { /** * Extract the Pulsar Function class from a function or archive. */ - public static String getFunctionClass(ClassLoader classLoader) throws IOException { - NarClassLoader ncl = (NarClassLoader) classLoader; - String configStr = ncl.getServiceDefinition(PULSAR_IO_SERVICE_NAME); - - FunctionDefinition conf = ObjectMapperFactory.getYamlMapper().reader().readValue(configStr, - FunctionDefinition.class); - if (StringUtils.isEmpty(conf.getFunctionClass())) { - throw new IOException( - String.format("The '%s' functionctor does not provide a function implementation", conf.getName())); - } + public static String getFunctionClass(File narFile) throws IOException { + return getFunctionDefinition(narFile).getFunctionClass(); + } - try { - // Try to load source class and check it implements Function interface - Class functionClass = ncl.loadClass(conf.getFunctionClass()); - if (!(Function.class.isAssignableFrom(functionClass))) { - throw new IOException( - "Class " + conf.getFunctionClass() + " does not implement interface " + Function.class - .getName()); - } - } catch (Throwable t) { - Exceptions.rethrowIOException(t); + public static FunctionDefinition getFunctionDefinition(File narFile) throws IOException { + return getPulsarIOServiceConfig(narFile, FunctionDefinition.class); + } + + public static T getPulsarIOServiceConfig(File narFile, Class valueType) throws IOException { + String filename = "META-INF/services/" + PULSAR_IO_SERVICE_NAME; + byte[] configEntry = ZipUtil.unpackEntry(narFile, filename); + if (configEntry != null) { + return ObjectMapperFactory.getYamlMapper().reader().readValue(configEntry, valueType); + } else { + return null; } + } - return conf.getFunctionClass(); + public static String getFunctionClass(NarClassLoader narClassLoader) throws IOException { + return getFunctionDefinition(narClassLoader).getFunctionClass(); } public static FunctionDefinition getFunctionDefinition(NarClassLoader narClassLoader) throws IOException { - String configStr = narClassLoader.getServiceDefinition(PULSAR_IO_SERVICE_NAME); - return ObjectMapperFactory.getYamlMapper().reader().readValue(configStr, FunctionDefinition.class); + return getPulsarIOServiceConfig(narClassLoader, FunctionDefinition.class); } - public static TreeMap searchForFunctions(String functionsDirectory) throws IOException { - return searchForFunctions(functionsDirectory, false); + public static T getPulsarIOServiceConfig(NarClassLoader narClassLoader, Class valueType) throws IOException { + return ObjectMapperFactory.getYamlMapper().reader() + .readValue(narClassLoader.getServiceDefinition(PULSAR_IO_SERVICE_NAME), valueType); } public static TreeMap searchForFunctions(String functionsDirectory, - boolean alwaysPopulatePath) throws IOException { + String narExtractionDirectory, + boolean enableClassloading) throws IOException { Path path = Paths.get(functionsDirectory).toAbsolutePath(); log.info("Searching for functions in {}", path); @@ -95,22 +91,12 @@ public static TreeMap searchForFunctions(String functio try (DirectoryStream stream = Files.newDirectoryStream(path, "*.nar")) { for (Path archive : stream) { try { - - NarClassLoader ncl = NarClassLoaderBuilder.builder() - .narFile(new File(archive.toString())) - .build(); - - FunctionArchive.FunctionArchiveBuilder functionArchiveBuilder = FunctionArchive.builder(); - FunctionDefinition cntDef = FunctionUtils.getFunctionDefinition(ncl); + FunctionDefinition cntDef = FunctionUtils.getFunctionDefinition(archive.toFile()); log.info("Found function {} from {}", cntDef, archive); - - functionArchiveBuilder.archivePath(archive); - - functionArchiveBuilder.classLoader(ncl); - functionArchiveBuilder.functionDefinition(cntDef); - - if (alwaysPopulatePath || !StringUtils.isEmpty(cntDef.getFunctionClass())) { - functions.put(cntDef.getName(), functionArchiveBuilder.build()); + if (!StringUtils.isEmpty(cntDef.getFunctionClass())) { + FunctionArchive functionArchive = + new FunctionArchive(archive, cntDef, narExtractionDirectory, enableClassloading); + functions.put(cntDef.getName(), functionArchive); } } catch (Throwable t) { log.warn("Failed to load function from {}", archive, t); diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/Connector.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/Connector.java index f1a03f4424ec6..5fcc22747c516 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/Connector.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/Connector.java @@ -20,17 +20,79 @@ import java.nio.file.Path; import java.util.List; -import lombok.Builder; -import lombok.Data; +import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.common.io.ConfigFieldDefinition; import org.apache.pulsar.common.io.ConnectorDefinition; +import org.apache.pulsar.functions.utils.FunctionFilePackage; +import org.apache.pulsar.functions.utils.ValidatableFunctionPackage; -@Builder -@Data -public class Connector { - private Path archivePath; +public class Connector implements AutoCloseable { + private final Path archivePath; + private final String narExtractionDirectory; + private final boolean enableClassloading; + private ValidatableFunctionPackage connectorFunctionPackage; private List sourceConfigFieldDefinitions; private List sinkConfigFieldDefinitions; - private ClassLoader classLoader; private ConnectorDefinition connectorDefinition; + private boolean closed; + + public Connector(Path archivePath, ConnectorDefinition connectorDefinition, String narExtractionDirectory, + boolean enableClassloading) { + this.archivePath = archivePath; + this.connectorDefinition = connectorDefinition; + this.narExtractionDirectory = narExtractionDirectory; + this.enableClassloading = enableClassloading; + } + + public Path getArchivePath() { + return archivePath; + } + + public synchronized ValidatableFunctionPackage getConnectorFunctionPackage() { + checkState(); + if (connectorFunctionPackage == null) { + connectorFunctionPackage = + new FunctionFilePackage(archivePath.toFile(), narExtractionDirectory, enableClassloading, + ConnectorDefinition.class); + } + return connectorFunctionPackage; + } + + private void checkState() { + if (closed) { + throw new IllegalStateException("Connector is already closed"); + } + } + + public synchronized List getSourceConfigFieldDefinitions() { + checkState(); + if (sourceConfigFieldDefinitions == null && !StringUtils.isEmpty(connectorDefinition.getSourceClass()) + && !StringUtils.isEmpty(connectorDefinition.getSourceConfigClass())) { + sourceConfigFieldDefinitions = ConnectorUtils.getConnectorConfigDefinition(getConnectorFunctionPackage(), + connectorDefinition.getSourceConfigClass()); + } + return sourceConfigFieldDefinitions; + } + + public synchronized List getSinkConfigFieldDefinitions() { + checkState(); + if (sinkConfigFieldDefinitions == null && !StringUtils.isEmpty(connectorDefinition.getSinkClass()) + && !StringUtils.isEmpty(connectorDefinition.getSinkConfigClass())) { + sinkConfigFieldDefinitions = ConnectorUtils.getConnectorConfigDefinition(getConnectorFunctionPackage(), + connectorDefinition.getSinkConfigClass()); + } + return sinkConfigFieldDefinitions; + } + + public ConnectorDefinition getConnectorDefinition() { + return connectorDefinition; + } + + @Override + public synchronized void close() throws Exception { + closed = true; + if (connectorFunctionPackage instanceof AutoCloseable) { + ((AutoCloseable) connectorFunctionPackage).close(); + } + } } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java index a814bf35548f3..df1310965f392 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java @@ -18,38 +18,31 @@ */ package org.apache.pulsar.functions.utils.io; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.File; import java.io.IOException; -import java.lang.annotation.Annotation; -import java.lang.reflect.Field; -import java.lang.reflect.Method; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.AbstractMap; -import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.TreeMap; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.stream.Collectors; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; +import net.bytebuddy.description.annotation.AnnotationDescription; +import net.bytebuddy.description.annotation.AnnotationValue; +import net.bytebuddy.description.field.FieldDescription; +import net.bytebuddy.description.method.MethodDescription; +import net.bytebuddy.description.type.TypeDefinition; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.common.io.ConfigFieldDefinition; import org.apache.pulsar.common.io.ConnectorDefinition; import org.apache.pulsar.common.nar.NarClassLoader; -import org.apache.pulsar.common.nar.NarClassLoaderBuilder; -import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.ObjectMapperFactory; -import org.apache.pulsar.common.util.Reflections; import org.apache.pulsar.functions.utils.Exceptions; +import org.apache.pulsar.functions.utils.ValidatableFunctionPackage; +import org.apache.pulsar.functions.utils.functions.FunctionUtils; import org.apache.pulsar.io.core.BatchSource; import org.apache.pulsar.io.core.Sink; import org.apache.pulsar.io.core.Source; @@ -76,7 +69,7 @@ public static String getIOSourceClass(NarClassLoader narClassLoader) throws IOEx Class sourceClass = narClassLoader.loadClass(conf.getSourceClass()); if (!(Source.class.isAssignableFrom(sourceClass) || BatchSource.class.isAssignableFrom(sourceClass))) { throw new IOException(String.format("Class %s does not implement interface %s or %s", - conf.getSourceClass(), Source.class.getName(), BatchSource.class.getName())); + conf.getSourceClass(), Source.class.getName(), BatchSource.class.getName())); } } catch (Throwable t) { Exceptions.rethrowIOException(t); @@ -109,32 +102,36 @@ public static String getIOSinkClass(NarClassLoader narClassLoader) throws IOExce return conf.getSinkClass(); } - public static ConnectorDefinition getConnectorDefinition(NarClassLoader narClassLoader) throws IOException { - String configStr = narClassLoader.getServiceDefinition(PULSAR_IO_SERVICE_NAME); + public static ConnectorDefinition getConnectorDefinition(File narFile) throws IOException { + return FunctionUtils.getPulsarIOServiceConfig(narFile, ConnectorDefinition.class); + } - return ObjectMapperFactory.getYamlMapper().reader().readValue(configStr, ConnectorDefinition.class); + public static ConnectorDefinition getConnectorDefinition(NarClassLoader narClassLoader) throws IOException { + return FunctionUtils.getPulsarIOServiceConfig(narClassLoader, ConnectorDefinition.class); } - public static List getConnectorConfigDefinition(ClassLoader classLoader, - String configClassName) throws Exception { + public static List getConnectorConfigDefinition( + ValidatableFunctionPackage connectorFunctionPackage, + String configClassName) { List retval = new LinkedList<>(); - Class configClass = classLoader.loadClass(configClassName); - for (Field field : Reflections.getAllFields(configClass)) { - if (java.lang.reflect.Modifier.isStatic(field.getModifiers())) { - // We dont want static fields + TypeDefinition configClass = connectorFunctionPackage.resolveType(configClassName); + + for (FieldDescription field : getAllFields(configClass)) { + if (field.isStatic()) { + // We don't want static fields continue; } - field.setAccessible(true); ConfigFieldDefinition configFieldDefinition = new ConfigFieldDefinition(); configFieldDefinition.setFieldName(field.getName()); - configFieldDefinition.setTypeName(field.getType().getName()); + configFieldDefinition.setTypeName(field.getType().getActualName()); Map attributes = new HashMap<>(); - for (Annotation annotation : field.getAnnotations()) { - if (annotation.annotationType().equals(FieldDoc.class)) { - FieldDoc fieldDoc = (FieldDoc) annotation; - for (Method method : FieldDoc.class.getDeclaredMethods()) { - Object value = method.invoke(fieldDoc); - attributes.put(method.getName(), value == null ? "" : value.toString()); + for (AnnotationDescription annotation : field.getDeclaredAnnotations()) { + if (annotation.getAnnotationType().represents(FieldDoc.class)) { + for (MethodDescription.InDefinedShape method : annotation.getAnnotationType() + .getDeclaredMethods()) { + AnnotationValue value = annotation.getValue(method.getName()); + attributes.put(method.getName(), + value == null || value.resolve() == null ? "" : value.resolve().toString()); } } } @@ -145,86 +142,42 @@ public static List getConnectorConfigDefinition(ClassLoad return retval; } + private static List getAllFields(TypeDefinition type) { + List fields = new LinkedList<>(); + fields.addAll(type.getDeclaredFields()); + + if (type.getSuperClass() != null) { + fields.addAll(getAllFields(type.getSuperClass())); + } + + return fields; + } + public static TreeMap searchForConnectors(String connectorsDirectory, - String narExtractionDirectory) throws IOException { + String narExtractionDirectory, + boolean enableClassloading) throws IOException { Path path = Paths.get(connectorsDirectory).toAbsolutePath(); log.info("Searching for connectors in {}", path); + TreeMap connectors = new TreeMap<>(); + if (!path.toFile().exists()) { log.warn("Connectors archive directory not found"); - return new TreeMap<>(); + return connectors; } - List archives = new ArrayList<>(); try (DirectoryStream stream = Files.newDirectoryStream(path, "*.nar")) { for (Path archive : stream) { - archives.add(archive); - } - } - if (archives.isEmpty()) { - return new TreeMap<>(); - } - - ExecutorService oneTimeExecutor = null; - try { - int nThreads = Math.min(Runtime.getRuntime().availableProcessors(), archives.size()); - log.info("Loading {} connector definitions with a thread pool of size {}", archives.size(), nThreads); - oneTimeExecutor = Executors.newFixedThreadPool(nThreads, - new ThreadFactoryBuilder().setNameFormat("connector-extraction-executor-%d").build()); - List>> futures = new ArrayList<>(); - for (Path archive : archives) { - CompletableFuture> future = CompletableFuture.supplyAsync(() -> - getConnectorDefinitionEntry(archive, narExtractionDirectory), oneTimeExecutor); - futures.add(future); - } - - FutureUtil.waitForAll(futures).join(); - return futures.stream() - .map(CompletableFuture::join) - .filter(entry -> entry != null) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a, b) -> a, TreeMap::new)); - } finally { - if (oneTimeExecutor != null) { - oneTimeExecutor.shutdown(); - } - } - } - - private static Map.Entry getConnectorDefinitionEntry(Path archive, - String narExtractionDirectory) { - try { - - NarClassLoader ncl = NarClassLoaderBuilder.builder() - .narFile(new File(archive.toString())) - .extractionDirectory(narExtractionDirectory) - .build(); - - Connector.ConnectorBuilder connectorBuilder = Connector.builder(); - ConnectorDefinition cntDef = ConnectorUtils.getConnectorDefinition(ncl); - log.info("Found connector {} from {}", cntDef, archive); - - connectorBuilder.archivePath(archive); - if (!StringUtils.isEmpty(cntDef.getSourceClass())) { - if (!StringUtils.isEmpty(cntDef.getSourceConfigClass())) { - connectorBuilder.sourceConfigFieldDefinitions( - ConnectorUtils.getConnectorConfigDefinition(ncl, - cntDef.getSourceConfigClass())); + try { + ConnectorDefinition cntDef = ConnectorUtils.getConnectorDefinition(archive.toFile()); + log.info("Found connector {} from {}", cntDef, archive); + Connector connector = new Connector(archive, cntDef, narExtractionDirectory, enableClassloading); + connectors.put(cntDef.getName(), connector); + } catch (Throwable t) { + log.warn("Failed to load connector from {}", archive, t); } } - - if (!StringUtils.isEmpty(cntDef.getSinkClass())) { - if (!StringUtils.isEmpty(cntDef.getSinkConfigClass())) { - connectorBuilder.sinkConfigFieldDefinitions( - ConnectorUtils.getConnectorConfigDefinition(ncl, cntDef.getSinkConfigClass())); - } - } - - connectorBuilder.classLoader(ncl); - connectorBuilder.connectorDefinition(cntDef); - return new AbstractMap.SimpleEntry(cntDef.getName(), connectorBuilder.build()); - } catch (Throwable t) { - log.warn("Failed to load connector from {}", archive, t); - return null; } + return connectors; } } diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java index 131f153b08d68..90fdd4da777d3 100644 --- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java @@ -30,16 +30,17 @@ import com.github.tomakehurst.wiremock.WireMockServer; import java.io.File; import java.util.Collection; +import java.util.concurrent.CompletableFuture; import lombok.Cleanup; +import net.bytebuddy.description.type.TypeDefinition; +import net.bytebuddy.pool.TypePool; import org.apache.pulsar.client.impl.MessageIdImpl; -import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.functions.api.Context; import org.apache.pulsar.functions.api.Function; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.functions.api.WindowContext; import org.apache.pulsar.functions.api.WindowFunction; import org.assertj.core.util.Files; -import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -47,41 +48,6 @@ * Unit test of {@link Exceptions}. */ public class FunctionCommonTest { - - @Test - public void testValidateLocalFileUrl() throws Exception { - String fileLocation = FutureUtil.class.getProtectionDomain().getCodeSource().getLocation().getPath(); - try { - // eg: fileLocation : /dir/fileName.jar (invalid) - FunctionCommon.extractClassLoader(fileLocation); - Assert.fail("should fail with invalid url: without protocol"); - } catch (IllegalArgumentException ie) { - // Ok.. expected exception - } - String fileLocationWithProtocol = "file://" + fileLocation; - // eg: fileLocation : file:///dir/fileName.jar (valid) - FunctionCommon.extractClassLoader(fileLocationWithProtocol); - // eg: fileLocation : file:/dir/fileName.jar (valid) - fileLocationWithProtocol = "file:" + fileLocation; - FunctionCommon.extractClassLoader(fileLocationWithProtocol); - } - - @Test - public void testValidateHttpFileUrl() throws Exception { - - String jarHttpUrl = "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar"; - FunctionCommon.extractClassLoader(jarHttpUrl); - - jarHttpUrl = "http://_invalidurl_.com"; - try { - // eg: fileLocation : /dir/fileName.jar (invalid) - FunctionCommon.extractClassLoader(jarHttpUrl); - Assert.fail("should fail with invalid url: without protocol"); - } catch (Exception ie) { - // Ok.. expected exception - } - } - @Test public void testDownloadFile() throws Exception { final String jarHttpUrl = "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar"; @@ -150,6 +116,14 @@ public Record process(String input, Context context) throws Exception { } }, false }, + { + new Function>() { + @Override + public CompletableFuture process(String input, Context context) throws Exception { + return null; + } + }, false + }, { new java.util.function.Function() { @Override @@ -166,6 +140,14 @@ public Record apply(String s) { } }, false }, + { + new java.util.function.Function>() { + @Override + public CompletableFuture apply(String s) { + return null; + } + }, false + }, { new WindowFunction() { @Override @@ -182,6 +164,14 @@ public Record process(Collection> input, WindowContext c } }, true }, + { + new WindowFunction>() { + @Override + public CompletableFuture process(Collection> input, WindowContext context) throws Exception { + return null; + } + }, true + }, { new java.util.function.Function, Integer>() { @Override @@ -197,15 +187,26 @@ public Record apply(Collection strings) { return null; } }, true + }, + { + new java.util.function.Function, CompletableFuture>() { + @Override + public CompletableFuture apply(Collection strings) { + return null; + } + }, true } }; } @Test(dataProvider = "function") public void testGetFunctionTypes(Object function, boolean isWindowConfigPresent) { - Class[] types = FunctionCommon.getFunctionTypes(function.getClass(), isWindowConfigPresent); + TypePool typePool = TypePool.Default.of(function.getClass().getClassLoader()); + TypeDefinition[] types = + FunctionCommon.getFunctionTypes(typePool.describe(function.getClass().getName()).resolve(), + isWindowConfigPresent); assertEquals(types.length, 2); - assertEquals(types[0], String.class); - assertEquals(types[1], Integer.class); + assertEquals(types[0].asErasure().getTypeName(), String.class.getName()); + assertEquals(types[1].asErasure().getTypeName(), Integer.class.getName()); } } diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java index 8f46199e8ffd5..954eef44a7366 100644 --- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java @@ -18,13 +18,24 @@ */ package org.apache.pulsar.functions.utils; +import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE; +import static org.apache.pulsar.common.functions.FunctionConfig.Runtime.PYTHON; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; import com.google.gson.Gson; - -import org.apache.pulsar.client.api.CompressionType; -import org.apache.pulsar.client.api.SubscriptionInitialPosition; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.client.api.CompressionType; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.impl.schema.JSONSchema; import org.apache.pulsar.common.functions.ConsumerConfig; import org.apache.pulsar.common.functions.FunctionConfig; @@ -32,28 +43,29 @@ import org.apache.pulsar.common.functions.Resources; import org.apache.pulsar.common.functions.WindowConfig; import org.apache.pulsar.common.util.Reflections; +import org.apache.pulsar.functions.api.Record; +import org.apache.pulsar.functions.api.WindowContext; +import org.apache.pulsar.functions.api.WindowFunction; import org.apache.pulsar.functions.api.utils.IdentityFunction; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.Function.FunctionDetails; import org.testng.annotations.Test; -import java.lang.reflect.Field; -import java.util.HashMap; -import java.util.Map; - -import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE; -import static org.apache.pulsar.common.functions.FunctionConfig.Runtime.PYTHON; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNull; -import static org.testng.Assert.assertThrows; -import static org.testng.Assert.assertTrue; - /** * Unit test of {@link Reflections}. */ @Slf4j public class FunctionConfigUtilsTest { + public static class WordCountWindowFunction implements WindowFunction { + @Override + public Void process(Collection> inputs, WindowContext context) throws Exception { + for (Record input : inputs) { + Arrays.asList(input.getValue().split("\\.")).forEach(word -> context.incrCounter(word, 1)); + } + return null; + } + } + @Test public void testAutoAckConvertFailed() { @@ -63,7 +75,7 @@ public void testAutoAckConvertFailed() { functionConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATMOST_ONCE); assertThrows(IllegalArgumentException.class, () -> { - FunctionConfigUtils.convert(functionConfig, (ClassLoader) null); + FunctionConfigUtils.convert(functionConfig); }); } @@ -99,7 +111,7 @@ public void testConvertBackFidelity() { producerConfig.setBatchBuilder("DEFAULT"); producerConfig.setCompressionType(CompressionType.ZLIB); functionConfig.setProducerConfig(producerConfig); - Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig, (ClassLoader) null); + Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig); FunctionConfig convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails); // add default resources @@ -119,7 +131,7 @@ public void testConvertWindow() { functionConfig.setNamespace("test-namespace"); functionConfig.setName("test-function"); functionConfig.setParallelism(1); - functionConfig.setClassName(IdentityFunction.class.getName()); + functionConfig.setClassName(WordCountWindowFunction.class.getName()); Map inputSpecs = new HashMap<>(); inputSpecs.put("test-input", ConsumerConfig.builder().isRegexPattern(true).serdeClassName("test-serde").build()); functionConfig.setInputSpecs(inputSpecs); @@ -141,7 +153,7 @@ public void testConvertWindow() { producerConfig.setBatchBuilder("KEY_BASED"); producerConfig.setCompressionType(CompressionType.SNAPPY); functionConfig.setProducerConfig(producerConfig); - Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig, (ClassLoader) null); + Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig); FunctionConfig convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails); // WindowsFunction guarantees convert to FunctionGuarantees. @@ -163,7 +175,7 @@ public void testConvertBatchBuilder() { FunctionConfig functionConfig = createFunctionConfig(); functionConfig.setBatchBuilder("KEY_BASED"); - Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig, (ClassLoader) null); + Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig); assertEquals(functionDetails.getSink().getProducerSpec().getBatchBuilder(), "KEY_BASED"); FunctionConfig convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails); @@ -519,7 +531,6 @@ private FunctionConfig createFunctionConfig() { functionConfig.setUserConfig(new HashMap<>()); functionConfig.setAutoAck(true); functionConfig.setTimeoutMs(2000L); - functionConfig.setWindowConfig(new WindowConfig().setWindowLengthCount(10)); functionConfig.setCleanupSubscription(true); functionConfig.setRuntimeFlags("-Dfoo=bar"); return functionConfig; @@ -553,7 +564,7 @@ public void testDisableForwardSourceMessageProperty() throws InvalidProtocolBuff config.setForwardSourceMessageProperty(true); FunctionConfigUtils.inferMissingArguments(config, false); assertNull(config.getForwardSourceMessageProperty()); - FunctionDetails details = FunctionConfigUtils.convert(config, FunctionConfigUtilsTest.class.getClassLoader()); + FunctionDetails details = FunctionConfigUtils.convert(config); assertFalse(details.getSink().getForwardSourceMessageProperty()); String detailsJson = "'" + JsonFormat.printer().omittingInsignificantWhitespace().print(details) + "'"; log.info("Function details : {}", detailsJson); @@ -640,7 +651,7 @@ public void testMergeDifferentOutputSchemaTypes() { @Test public void testPoolMessages() { FunctionConfig functionConfig = createFunctionConfig(); - Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig, (ClassLoader) null); + Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig); assertFalse(functionDetails.getSource().getInputSpecsMap().get("test-input").getPoolMessages()); FunctionConfig convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails); assertFalse(convertedConfig.getInputSpecs().get("test-input").isPoolMessages()); @@ -650,7 +661,7 @@ public void testPoolMessages() { .poolMessages(true).build()); functionConfig.setInputSpecs(inputSpecs); - functionDetails = FunctionConfigUtils.convert(functionConfig, (ClassLoader) null); + functionDetails = FunctionConfigUtils.convert(functionConfig); assertTrue(functionDetails.getSource().getInputSpecsMap().get("test-input").getPoolMessages()); convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails); diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java index 8ac9b61e3f60f..14cd77f60ff95 100644 --- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java @@ -18,37 +18,38 @@ */ package org.apache.pulsar.functions.utils; +import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE; +import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.ATMOST_ONCE; +import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; import com.google.common.collect.Lists; import com.google.gson.Gson; +import java.io.IOException; +import java.lang.reflect.Field; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import lombok.Data; import lombok.NoArgsConstructor; import lombok.experimental.Accessors; import org.apache.pulsar.common.functions.ConsumerConfig; import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.functions.Resources; +import org.apache.pulsar.common.io.ConnectorDefinition; import org.apache.pulsar.common.io.SinkConfig; import org.apache.pulsar.config.validation.ConfigValidationAnnotations; -import org.apache.pulsar.functions.api.utils.IdentityFunction; +import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.functions.proto.Function; +import org.apache.pulsar.io.core.Sink; +import org.apache.pulsar.io.core.SinkContext; import org.testng.annotations.Test; -import java.io.IOException; -import java.lang.reflect.Field; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE; -import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.ATMOST_ONCE; -import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertThrows; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.expectThrows; - /** * Unit test of {@link SinkConfigUtilsTest}. */ @@ -62,6 +63,27 @@ public static class TestSinkConfig { private String configParameter; } + + public static class NopSink implements Sink { + + @Override + public void open(Map config, SinkContext sinkContext) throws Exception { + + } + + @Override + public void write(Record record) throws Exception { + + } + + @Override + public void close() throws Exception { + + } + } + + + @Test public void testAutoAckConvertFailed() throws IOException { @@ -521,7 +543,7 @@ private SinkConfig createSinkConfig() { sinkConfig.setNamespace("test-namespace"); sinkConfig.setName("test-sink"); sinkConfig.setParallelism(1); - sinkConfig.setClassName(IdentityFunction.class.getName()); + sinkConfig.setClassName(NopSink.class.getName()); Map inputSpecs = new HashMap<>(); inputSpecs.put("test-input", ConsumerConfig.builder().isRegexPattern(true).serdeClassName("test-serde").build()); sinkConfig.setInputSpecs(inputSpecs); @@ -577,13 +599,16 @@ public void testAllowDisableSinkTimeout() { SinkConfig sinkConfig = createSinkConfig(); sinkConfig.setInputSpecs(null); sinkConfig.setTopicsPattern("my-topic-*"); - SinkConfigUtils.validateAndExtractDetails(sinkConfig, this.getClass().getClassLoader(), null, + LoadedFunctionPackage validatableFunction = + new LoadedFunctionPackage(this.getClass().getClassLoader(), ConnectorDefinition.class); + + SinkConfigUtils.validateAndExtractDetails(sinkConfig, validatableFunction, null, true); sinkConfig.setTimeoutMs(null); - SinkConfigUtils.validateAndExtractDetails(sinkConfig, this.getClass().getClassLoader(), null, + SinkConfigUtils.validateAndExtractDetails(sinkConfig, validatableFunction, null, true); sinkConfig.setTimeoutMs(0L); - SinkConfigUtils.validateAndExtractDetails(sinkConfig, this.getClass().getClassLoader(), null, + SinkConfigUtils.validateAndExtractDetails(sinkConfig, validatableFunction, null, true); } } diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml index c470af18890f6..8764c29434a92 100644 --- a/pulsar-functions/worker/pom.xml +++ b/pulsar-functions/worker/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-functions 3.2.0-SNAPSHOT .. - pulsar-functions-worker Pulsar Functions :: Worker - - ${project.groupId} pulsar-broker-common ${project.version} - ${project.groupId} pulsar-functions-runtime ${project.version} - ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} pulsar-client-admin-original ${project.version} - org.glassfish.jersey.media jersey-media-json-jackson - jakarta.activation jakarta.activation-api - org.glassfish.jersey.core jersey-server - org.glassfish.jersey.containers jersey-container-servlet - org.glassfish.jersey.containers jersey-container-servlet-core - org.glassfish.jersey.media jersey-media-multipart - org.eclipse.jetty jetty-server - org.eclipse.jetty jetty-alpn-conscrypt-server - org.eclipse.jetty jetty-servlet - org.eclipse.jetty jetty-servlets - org.apache.distributedlog distributedlog-core @@ -118,17 +100,14 @@ - org.apache.bookkeeper bookkeeper-server - commons-io commons-io - ${project.groupId} pulsar-docs-tools @@ -140,7 +119,6 @@ - io.swagger swagger-core @@ -156,14 +134,11 @@ - io.prometheus simpleclient_jetty - - ${project.groupId} pulsar-io-cassandra @@ -171,7 +146,6 @@ pom test - ${project.groupId} pulsar-io-twitter @@ -179,7 +153,6 @@ pom test - ${project.groupId} pulsar-functions-api-examples @@ -187,7 +160,6 @@ pom test - ${project.groupId} pulsar-functions-api-examples-builtin @@ -195,9 +167,7 @@ pom test - - @@ -217,7 +187,6 @@ - maven-dependency-plugin @@ -270,7 +239,6 @@ - org.apache.maven.plugins maven-surefire-plugin @@ -286,7 +254,6 @@ - org.apache.maven.plugins @@ -334,5 +301,4 @@ - diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java index 03c6eb7921840..389051fce4217 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java @@ -70,6 +70,7 @@ import org.apache.pulsar.functions.utils.Actions; import org.apache.pulsar.functions.utils.FunctionCommon; import org.apache.pulsar.functions.utils.SourceConfigUtils; +import org.apache.pulsar.functions.utils.ValidatableFunctionPackage; import org.apache.pulsar.functions.utils.io.Connector; @Data @@ -82,18 +83,21 @@ public class FunctionActioner { private final ConnectorsManager connectorsManager; private final FunctionsManager functionsManager; private final PulsarAdmin pulsarAdmin; + private final PackageUrlValidator packageUrlValidator; public FunctionActioner(WorkerConfig workerConfig, RuntimeFactory runtimeFactory, Namespace dlogNamespace, ConnectorsManager connectorsManager, - FunctionsManager functionsManager, PulsarAdmin pulsarAdmin) { + FunctionsManager functionsManager, PulsarAdmin pulsarAdmin, + PackageUrlValidator packageUrlValidator) { this.workerConfig = workerConfig; this.runtimeFactory = runtimeFactory; this.dlogNamespace = dlogNamespace; this.connectorsManager = connectorsManager; this.functionsManager = functionsManager; this.pulsarAdmin = pulsarAdmin; + this.packageUrlValidator = packageUrlValidator; } @@ -151,6 +155,9 @@ private String getPackageFile(FunctionMetaData functionMetaData, FunctionDetails boolean isPkgUrlProvided = isFunctionPackageUrlSupported(packagePath); String packageFile; if (isPkgUrlProvided && packagePath.startsWith(FILE)) { + if (!packageUrlValidator.isValidPackageUrl(componentType, packagePath)) { + throw new IllegalArgumentException("Package URL " + packagePath + " is not valid"); + } URL url = new URL(packagePath); File pkgFile = new File(url.toURI()); packageFile = pkgFile.getAbsolutePath(); @@ -167,7 +174,7 @@ private String getPackageFile(FunctionMetaData functionMetaData, FunctionDetails pkgDir, new File(getDownloadFileName(functionMetaData.getFunctionDetails(), pkgLocation)).getName()); - downloadFile(pkgFile, isPkgUrlProvided, functionMetaData, instanceId, pkgLocation); + downloadFile(pkgFile, isPkgUrlProvided, functionMetaData, instanceId, pkgLocation, componentType); packageFile = pkgFile.getAbsolutePath(); } return packageFile; @@ -226,7 +233,8 @@ InstanceConfig createInstanceConfig(FunctionDetails functionDetails, Function.Fu } private void downloadFile(File pkgFile, boolean isPkgUrlProvided, FunctionMetaData functionMetaData, - int instanceId, Function.PackageLocationMetaData pkgLocation) + int instanceId, Function.PackageLocationMetaData pkgLocation, + FunctionDetails.ComponentType componentType) throws IOException, PulsarAdminException { FunctionDetails details = functionMetaData.getFunctionDetails(); @@ -251,6 +259,9 @@ private void downloadFile(File pkgFile, boolean isPkgUrlProvided, FunctionMetaDa downloadFromHttp ? pkgLocationPath : pkgLocation); if (downloadFromHttp) { + if (!packageUrlValidator.isValidPackageUrl(componentType, pkgLocationPath)) { + throw new IllegalArgumentException("Package URL " + pkgLocationPath + " is not valid"); + } FunctionCommon.downloadFromHttpUrl(pkgLocationPath, tempPkgFile); } else if (downloadFromPackageManagementService) { getPulsarAdmin().packages().download(pkgLocationPath, tempPkgFile.getPath()); @@ -527,7 +538,7 @@ private File getBuiltinArchive(FunctionDetails.ComponentType componentType, Func builder.setClassName(sourceClass); functionDetails.setSource(builder); - fillSourceTypeClass(functionDetails, connector.getClassLoader(), sourceClass); + fillSourceTypeClass(functionDetails, connector.getConnectorFunctionPackage(), sourceClass); return archive; } } @@ -543,7 +554,7 @@ private File getBuiltinArchive(FunctionDetails.ComponentType componentType, Func builder.setClassName(sinkClass); functionDetails.setSink(builder); - fillSinkTypeClass(functionDetails, connector.getClassLoader(), sinkClass); + fillSinkTypeClass(functionDetails, connector.getConnectorFunctionPackage(), sinkClass); return archive; } } @@ -557,8 +568,8 @@ private File getBuiltinArchive(FunctionDetails.ComponentType componentType, Func } private void fillSourceTypeClass(FunctionDetails.Builder functionDetails, - ClassLoader narClassLoader, String className) throws ClassNotFoundException { - String typeArg = getSourceType(className, narClassLoader).getName(); + ValidatableFunctionPackage functionPackage, String className) { + String typeArg = getSourceType(className, functionPackage.getTypePool()).asErasure().getName(); SourceSpec.Builder sourceBuilder = SourceSpec.newBuilder(functionDetails.getSource()); sourceBuilder.setTypeClassName(typeArg); @@ -573,8 +584,8 @@ private void fillSourceTypeClass(FunctionDetails.Builder functionDetails, } private void fillSinkTypeClass(FunctionDetails.Builder functionDetails, - ClassLoader narClassLoader, String className) throws ClassNotFoundException { - String typeArg = getSinkType(className, narClassLoader).getName(); + ValidatableFunctionPackage functionPackage, String className) { + String typeArg = getSinkType(className, functionPackage.getTypePool()).asErasure().getName(); SinkSpec.Builder sinkBuilder = SinkSpec.newBuilder(functionDetails.getSink()); sinkBuilder.setTypeClassName(typeArg); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java index 8e6725e93af91..b6e2bbb1ca0f8 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java @@ -220,7 +220,8 @@ public FunctionRuntimeManager(WorkerConfig workerConfig, PulsarWorkerService wor functionAuthProvider, runtimeCustomizer); this.functionActioner = new FunctionActioner(this.workerConfig, runtimeFactory, - dlogNamespace, connectorsManager, functionsManager, workerService.getBrokerAdmin()); + dlogNamespace, connectorsManager, functionsManager, workerService.getBrokerAdmin(), + workerService.getPackageUrlValidator()); this.membershipManager = membershipManager; this.functionMetaDataManager = functionMetaDataManager; diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PackageUrlValidator.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PackageUrlValidator.java new file mode 100644 index 0000000000000..2a8fe8dddb153 --- /dev/null +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PackageUrlValidator.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.worker; + +import java.net.URI; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import org.apache.pulsar.functions.proto.Function; + +/** + * Validates package URLs for functions and connectors. + * Validates that the package URL is either a file in the connectors or functions directory + * when referencing connector or function files is enabled, or matches one of the additional url patterns. + */ +public class PackageUrlValidator { + private final Path connectionsDirectory; + private final Path functionsDirectory; + private final List additionalConnectionsPatterns; + private final List additionalFunctionsPatterns; + + public PackageUrlValidator(WorkerConfig workerConfig) { + this.connectionsDirectory = resolveDirectory(workerConfig.getEnableReferencingConnectorDirectoryFiles(), + workerConfig.getConnectorsDirectory()); + this.functionsDirectory = resolveDirectory(workerConfig.getEnableReferencingFunctionsDirectoryFiles(), + workerConfig.getFunctionsDirectory()); + this.additionalConnectionsPatterns = + compilePatterns(workerConfig.getAdditionalEnabledConnectorUrlPatterns()); + this.additionalFunctionsPatterns = + compilePatterns(workerConfig.getAdditionalEnabledFunctionsUrlPatterns()); + } + + private static Path resolveDirectory(Boolean enabled, String directory) { + return enabled != null && enabled + ? Path.of(directory).normalize().toAbsolutePath() : null; + } + + private static List compilePatterns(List additionalPatterns) { + return additionalPatterns != null ? additionalPatterns.stream().map(Pattern::compile).collect( + Collectors.toList()) : Collections.emptyList(); + } + + boolean isValidFunctionsPackageUrl(URI functionPkgUrl) { + return doesMatch(functionPkgUrl, functionsDirectory, additionalFunctionsPatterns); + } + + boolean isValidConnectionsPackageUrl(URI functionPkgUrl) { + return doesMatch(functionPkgUrl, connectionsDirectory, additionalConnectionsPatterns); + } + + private boolean doesMatch(URI functionPkgUrl, Path directory, List patterns) { + if (directory != null && "file".equals(functionPkgUrl.getScheme())) { + Path filePath = Path.of(functionPkgUrl.getPath()).normalize().toAbsolutePath(); + if (filePath.startsWith(directory)) { + return true; + } + } + String functionPkgUrlString = functionPkgUrl.normalize().toString(); + for (Pattern pattern : patterns) { + if (pattern.matcher(functionPkgUrlString).matches()) { + return true; + } + } + return false; + } + + public boolean isValidPackageUrl(Function.FunctionDetails.ComponentType componentType, String functionPkgUrl) { + URI uri = URI.create(functionPkgUrl); + if (componentType == null) { + // if component type is not specified, we need to check both functions and connections + return isValidFunctionsPackageUrl(uri) || isValidConnectionsPackageUrl(uri); + } + switch (componentType) { + case FUNCTION: + return isValidFunctionsPackageUrl(uri); + case SINK: + case SOURCE: + return isValidConnectionsPackageUrl(uri); + default: + throw new IllegalArgumentException("Unknown component type: " + componentType); + } + } +} diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java index 16cf778e07290..7fff2830d1f17 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java @@ -118,7 +118,8 @@ public interface PulsarClientCreator { private Sinks sinks; private Sources sources; private Workers workers; - + @Getter + private PackageUrlValidator packageUrlValidator; private final PulsarClientCreator clientCreator; private StateStoreProvider stateStoreProvider; @@ -196,6 +197,7 @@ public void init(WorkerConfig workerConfig, this.sinks = new SinksImpl(() -> PulsarWorkerService.this); this.sources = new SourcesImpl(() -> PulsarWorkerService.this); this.workers = new WorkerImpl(() -> PulsarWorkerService.this); + this.packageUrlValidator = new PackageUrlValidator(workerConfig); } @Override @@ -222,7 +224,7 @@ private static URI initializeStandaloneWorkerService(PulsarClientCreator clientC log.warn("Retry to connect to Pulsar service at {}", workerConfig.getPulsarWebServiceUrl()); if (retries >= maxRetries) { log.error("Failed to connect to Pulsar service at {} after {} attempts", - workerConfig.getPulsarFunctionsNamespace(), maxRetries); + workerConfig.getPulsarFunctionsNamespace(), maxRetries, e); throw e; } retries++; @@ -659,6 +661,14 @@ public void stop() { if (null != stateStoreProvider) { stateStoreProvider.close(); } + + if (null != functionsManager) { + functionsManager.close(); + } + + if (null != connectorsManager) { + connectorsManager.close(); + } } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java index b175a7f275e71..ba87713d3c13e 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java @@ -74,6 +74,7 @@ import org.apache.pulsar.common.policies.data.FunctionStatsImpl; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.RestException; +import org.apache.pulsar.functions.api.state.StateValue; import org.apache.pulsar.functions.instance.InstanceUtils; import org.apache.pulsar.functions.instance.state.DefaultStateStore; import org.apache.pulsar.functions.proto.Function; @@ -88,6 +89,7 @@ import org.apache.pulsar.functions.utils.FunctionCommon; import org.apache.pulsar.functions.utils.FunctionConfigUtils; import org.apache.pulsar.functions.utils.FunctionMetaDataUtils; +import org.apache.pulsar.functions.utils.ValidatableFunctionPackage; import org.apache.pulsar.functions.utils.functions.FunctionArchive; import org.apache.pulsar.functions.utils.io.Connector; import org.apache.pulsar.functions.worker.FunctionMetaDataManager; @@ -480,7 +482,7 @@ public void deregisterFunction(final String tenant, try { worker().getStateStoreProvider().cleanUp(tenant, namespace, componentName); } catch (Throwable e) { - log.error("failed to clean up the state store for {}/{}/{}", tenant, namespace, componentName); + log.error("failed to clean up the state store for {}/{}/{}", tenant, namespace, componentName, e); } } } @@ -1053,7 +1055,7 @@ public String triggerFunction(final String tenant, try { worker().getBrokerAdmin().topics().getSubscriptions(inputTopicToWrite); } catch (PulsarAdminException e) { - log.error("Function in trigger function is not ready @ /{}/{}/{}", tenant, namespace, functionName); + log.error("Function in trigger function is not ready @ /{}/{}/{}", tenant, namespace, functionName, e); throw new RestException(Status.BAD_REQUEST, "Function in trigger function is not ready"); } String outputTopic = functionMetaData.getFunctionDetails().getSink().getTopic(); @@ -1149,26 +1151,40 @@ public FunctionState getFunctionState(final String tenant, throw new RestException(Status.BAD_REQUEST, e.getMessage()); } + FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); + if (!functionMetaDataManager.containsFunction(tenant, namespace, functionName)) { + log.warn("getFunctionState does not exist @ /{}/{}/{}", tenant, namespace, functionName); + throw new RestException(Status.NOT_FOUND, String.format("'%s' is not found", functionName)); + } + try { DefaultStateStore store = worker().getStateStoreProvider().getStateStore(tenant, namespace, functionName); - ByteBuffer buf = store.get(key); - if (buf == null) { + StateValue value = store.getStateValue(key); + if (value == null) { throw new RestException(Status.NOT_FOUND, "key '" + key + "' doesn't exist."); } + byte[] data = value.getValue(); + if (data == null) { + throw new RestException(Status.NOT_FOUND, "key '" + key + "' doesn't exist."); + } + + ByteBuffer buf = ByteBuffer.wrap(data); - // try to parse the state as a long - // but even if it can be parsed as a long, this number may not be the actual state, - // so we will always return a `stringValue` or `bytesValue` with the number value Long number = null; if (buf.remaining() == Long.BYTES) { number = buf.getLong(); } + if (Boolean.TRUE.equals(value.getIsNumber())) { + return new FunctionState(key, null, null, number, value.getVersion()); + } - if (Utf8.isWellFormed(buf.array())) { - return new FunctionState(key, new String(buf.array(), UTF_8), null, number, null); + if (Utf8.isWellFormed(data)) { + return new FunctionState(key, new String(data, UTF_8), null, number, value.getVersion()); } else { - return new FunctionState(key, null, buf.array(), number, null); + return new FunctionState(key, null, data, number, value.getVersion()); } + } catch (RestException e) { + throw e; } catch (Throwable e) { log.error("Error while getFunctionState request @ /{}/{}/{}/{}", tenant, namespace, functionName, key, e); @@ -1210,18 +1226,26 @@ public void putFunctionState(final String tenant, throw new RestException(Status.BAD_REQUEST, e.getMessage()); } + FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); + if (!functionMetaDataManager.containsFunction(tenant, namespace, functionName)) { + log.warn("putFunctionState does not exist @ /{}/{}/{}", tenant, namespace, functionName); + throw new RestException(Status.NOT_FOUND, String.format("'%s' is not found", functionName)); + } + try { DefaultStateStore store = worker().getStateStoreProvider().getStateStore(tenant, namespace, functionName); ByteBuffer data; - if (StringUtils.isNotEmpty(state.getStringValue())) { - data = ByteBuffer.wrap(state.getStringValue().getBytes(UTF_8)); - } else if (state.getByteValue() != null) { - data = ByteBuffer.wrap(state.getByteValue()); - } else if (state.getNumberValue() != null) { - data = ByteBuffer.allocate(Long.BYTES); - data.putLong(state.getNumberValue()); + if (state.getByteValue() == null || state.getByteValue().length == 0) { + if (state.getStringValue() != null) { + data = ByteBuffer.wrap(state.getStringValue().getBytes(UTF_8)); + } else if (state.getNumberValue() != null) { + data = ByteBuffer.allocate(Long.BYTES); + data.putLong(state.getNumberValue()); + } else { + throw new IllegalArgumentException("Invalid state value"); + } } else { - throw new IllegalArgumentException("Invalid state value"); + data = ByteBuffer.wrap(state.getByteValue()); } store.put(key, data); } catch (Throwable e) { @@ -1310,11 +1334,17 @@ private StreamingOutput getStreamingOutput(String pkgPath) { private StreamingOutput getStreamingOutput(String pkgPath, FunctionDetails.ComponentType componentType) { return output -> { if (pkgPath.startsWith(Utils.HTTP)) { + if (!worker().getPackageUrlValidator().isValidPackageUrl(componentType, pkgPath)) { + throw new IllegalArgumentException("Invalid package url: " + pkgPath); + } URL url = URI.create(pkgPath).toURL(); try (InputStream inputStream = url.openStream()) { IOUtils.copy(inputStream, output); } } else if (pkgPath.startsWith(Utils.FILE)) { + if (!worker().getPackageUrlValidator().isValidPackageUrl(componentType, pkgPath)) { + throw new IllegalArgumentException("Invalid package url: " + pkgPath); + } URI url = URI.create(pkgPath); File file = new File(url.getPath()); Files.copy(file.toPath(), output); @@ -1723,12 +1753,6 @@ private void internalProcessFunctionRequest(final String tenant, final String na } } - protected ClassLoader getClassLoaderFromPackage(String className, - File packageFile, - String narExtractionDirectory) { - return FunctionCommon.getClassLoaderFromPackage(componentType, className, packageFile, narExtractionDirectory); - } - static File downloadPackageFile(PulsarWorkerService worker, String packageName) throws IOException, PulsarAdminException { Path tempDirectory; @@ -1744,12 +1768,17 @@ static File downloadPackageFile(PulsarWorkerService worker, String packageName) return file; } - protected File getPackageFile(String functionPkgUrl, String existingPackagePath, InputStream uploadedInputStream) + protected File getPackageFile(FunctionDetails.ComponentType componentType, String functionPkgUrl, + String existingPackagePath, InputStream uploadedInputStream) throws IOException, PulsarAdminException { File componentPackageFile = null; if (isNotBlank(functionPkgUrl)) { - componentPackageFile = getPackageFile(functionPkgUrl); + componentPackageFile = getPackageFile(componentType, functionPkgUrl); } else if (existingPackagePath.startsWith(Utils.FILE) || existingPackagePath.startsWith(Utils.HTTP)) { + if (!worker().getPackageUrlValidator().isValidPackageUrl(componentType, functionPkgUrl)) { + throw new IllegalArgumentException("Function Package url is not valid." + + "supported url (http/https/file)"); + } try { componentPackageFile = FunctionCommon.extractFileFromPkgURL(existingPackagePath); } catch (Exception e) { @@ -1757,6 +1786,8 @@ protected File getPackageFile(String functionPkgUrl, String existingPackagePath, + "when getting %s package from %s", e.getMessage(), ComponentTypeUtils.toString(componentType), functionPkgUrl)); } + } else if (Utils.hasPackageTypePrefix(existingPackagePath)) { + componentPackageFile = getPackageFile(componentType, existingPackagePath); } else if (uploadedInputStream != null) { componentPackageFile = WorkerUtils.dumpToTmpFile(uploadedInputStream); } else if (!existingPackagePath.startsWith(Utils.BUILTIN)) { @@ -1774,15 +1805,16 @@ protected File getPackageFile(String functionPkgUrl, String existingPackagePath, return componentPackageFile; } - protected File downloadPackageFile(String packageName) throws IOException, PulsarAdminException { - return downloadPackageFile(worker(), packageName); - } - - protected File getPackageFile(String functionPkgUrl) throws IOException, PulsarAdminException { + protected File getPackageFile(FunctionDetails.ComponentType componentType, String functionPkgUrl) + throws IOException, PulsarAdminException { if (Utils.hasPackageTypePrefix(functionPkgUrl)) { - return downloadPackageFile(functionPkgUrl); + if (!worker().getWorkerConfig().isFunctionsWorkerEnablePackageManagement()) { + throw new IllegalStateException("Function Package management service is disabled. " + + "Please enable it to use " + functionPkgUrl); + } + return downloadPackageFile(worker(), functionPkgUrl); } else { - if (!Utils.isFunctionPackageUrlSupported(functionPkgUrl)) { + if (!worker().getPackageUrlValidator().isValidPackageUrl(componentType, functionPkgUrl)) { throw new IllegalArgumentException("Function Package url is not valid." + "supported url (http/https/file)"); } @@ -1796,7 +1828,7 @@ protected File getPackageFile(String functionPkgUrl) throws IOException, PulsarA } } - protected ClassLoader getBuiltinFunctionClassLoader(String archive) { + protected ValidatableFunctionPackage getBuiltinFunctionPackage(String archive) { if (!StringUtils.isEmpty(archive)) { if (archive.startsWith(org.apache.pulsar.common.functions.Utils.BUILTIN)) { archive = archive.replaceFirst("^builtin://", ""); @@ -1806,7 +1838,7 @@ protected ClassLoader getBuiltinFunctionClassLoader(String archive) { if (function == null) { throw new IllegalArgumentException("Built-in " + componentType + " is not available"); } - return function.getClassLoader(); + return function.getFunctionPackage(); } } return null; diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java index 078c47524f8e9..4cbd7c8cbcb12 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java @@ -47,7 +47,6 @@ import org.apache.pulsar.common.functions.WorkerInfo; import org.apache.pulsar.common.policies.data.ExceptionInformation; import org.apache.pulsar.common.policies.data.FunctionStatus; -import org.apache.pulsar.common.util.ClassLoaderUtils; import org.apache.pulsar.common.util.RestException; import org.apache.pulsar.functions.auth.FunctionAuthData; import org.apache.pulsar.functions.instance.InstanceUtils; @@ -55,11 +54,14 @@ import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.utils.ComponentTypeUtils; import org.apache.pulsar.functions.utils.FunctionConfigUtils; +import org.apache.pulsar.functions.utils.FunctionFilePackage; import org.apache.pulsar.functions.utils.FunctionMetaDataUtils; +import org.apache.pulsar.functions.utils.ValidatableFunctionPackage; import org.apache.pulsar.functions.utils.functions.FunctionArchive; import org.apache.pulsar.functions.worker.FunctionMetaDataManager; import org.apache.pulsar.functions.worker.FunctionsManager; import org.apache.pulsar.functions.worker.PulsarWorkerService; +import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerUtils; import org.apache.pulsar.functions.worker.service.api.Functions; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; @@ -142,7 +144,7 @@ public void registerFunction(final String tenant, // validate parameters try { if (isNotBlank(functionPkgUrl)) { - componentPackageFile = getPackageFile(functionPkgUrl); + componentPackageFile = getPackageFile(componentType, functionPkgUrl); functionDetails = validateUpdateRequestParams(tenant, namespace, functionName, functionConfig, componentPackageFile); } else { @@ -167,7 +169,7 @@ public void registerFunction(final String tenant, worker().getFunctionRuntimeManager().getRuntimeFactory().doAdmissionChecks(functionDetails); } catch (Exception e) { log.error("{} {}/{}/{} cannot be admitted by the runtime factory", - ComponentTypeUtils.toString(componentType), tenant, namespace, functionName); + ComponentTypeUtils.toString(componentType), tenant, namespace, functionName, e); throw new RestException(Response.Status.BAD_REQUEST, String.format("%s %s cannot be admitted:- %s", ComponentTypeUtils.toString(componentType), functionName, e.getMessage())); } @@ -303,6 +305,7 @@ public void updateFunction(final String tenant, // validate parameters try { componentPackageFile = getPackageFile( + componentType, functionPkgUrl, existingComponent.getPackageLocation().getPackagePath(), uploadedInputStream); @@ -324,7 +327,7 @@ public void updateFunction(final String tenant, worker().getFunctionRuntimeManager().getRuntimeFactory().doAdmissionChecks(functionDetails); } catch (Exception e) { log.error("Updated {} {}/{}/{} cannot be submitted to runtime factory", - ComponentTypeUtils.toString(componentType), tenant, namespace, functionName); + ComponentTypeUtils.toString(componentType), tenant, namespace, functionName, e); throw new RestException(Response.Status.BAD_REQUEST, String.format("%s %s cannot be admitted:- %s", ComponentTypeUtils.toString(componentType), functionName, e.getMessage())); } @@ -732,11 +735,12 @@ private Function.FunctionDetails validateUpdateRequestParams(final String tenant functionConfig.setTenant(tenant); functionConfig.setNamespace(namespace); functionConfig.setName(componentName); + WorkerConfig workerConfig = worker().getWorkerConfig(); FunctionConfigUtils.inferMissingArguments( - functionConfig, worker().getWorkerConfig().isForwardSourceMessageProperty()); + functionConfig, workerConfig.isForwardSourceMessageProperty()); String archive = functionConfig.getJar(); - ClassLoader classLoader = null; + ValidatableFunctionPackage functionPackage = null; // check if function is builtin and extract classloader if (!StringUtils.isEmpty(archive)) { if (archive.startsWith(org.apache.pulsar.common.functions.Utils.BUILTIN)) { @@ -749,35 +753,38 @@ private Function.FunctionDetails validateUpdateRequestParams(final String tenant if (function == null) { throw new IllegalArgumentException(String.format("No Function %s found", archive)); } - classLoader = function.getClassLoader(); + functionPackage = function.getFunctionPackage(); } } - boolean shouldCloseClassLoader = false; + boolean shouldCloseFunctionPackage = false; try { - if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) { // if function is not builtin, attempt to extract classloader from package file if it exists - if (classLoader == null && componentPackageFile != null) { - classLoader = getClassLoaderFromPackage(functionConfig.getClassName(), - componentPackageFile, worker().getWorkerConfig().getNarExtractionDirectory()); - shouldCloseClassLoader = true; + if (functionPackage == null && componentPackageFile != null) { + functionPackage = + new FunctionFilePackage(componentPackageFile, workerConfig.getNarExtractionDirectory(), + workerConfig.getEnableClassloadingOfExternalFiles(), FunctionDefinition.class); + shouldCloseFunctionPackage = true; } - if (classLoader == null) { + if (functionPackage == null) { throw new IllegalArgumentException("Function package is not provided"); } FunctionConfigUtils.ExtractedFunctionDetails functionDetails = FunctionConfigUtils.validateJavaFunction( - functionConfig, classLoader); + functionConfig, functionPackage); return FunctionConfigUtils.convert(functionConfig, functionDetails); } else { - classLoader = FunctionConfigUtils.validate(functionConfig, componentPackageFile); - shouldCloseClassLoader = true; - return FunctionConfigUtils.convert(functionConfig, classLoader); + FunctionConfigUtils.validateNonJavaFunction(functionConfig); + return FunctionConfigUtils.convert(functionConfig); } } finally { - if (shouldCloseClassLoader) { - ClassLoaderUtils.closeClassLoader(classLoader); + if (shouldCloseFunctionPackage && functionPackage instanceof AutoCloseable) { + try { + ((AutoCloseable) functionPackage).close(); + } catch (Exception e) { + log.error("Failed to close function file", e); + } } } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java index c382ec9e01b35..6b8b41e5a8e5b 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java @@ -47,18 +47,20 @@ import org.apache.pulsar.common.io.SinkConfig; import org.apache.pulsar.common.policies.data.ExceptionInformation; import org.apache.pulsar.common.policies.data.SinkStatus; -import org.apache.pulsar.common.util.ClassLoaderUtils; import org.apache.pulsar.common.util.RestException; import org.apache.pulsar.functions.auth.FunctionAuthData; import org.apache.pulsar.functions.instance.InstanceUtils; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.utils.ComponentTypeUtils; +import org.apache.pulsar.functions.utils.FunctionFilePackage; import org.apache.pulsar.functions.utils.FunctionMetaDataUtils; import org.apache.pulsar.functions.utils.SinkConfigUtils; +import org.apache.pulsar.functions.utils.ValidatableFunctionPackage; import org.apache.pulsar.functions.utils.io.Connector; import org.apache.pulsar.functions.worker.FunctionMetaDataManager; import org.apache.pulsar.functions.worker.PulsarWorkerService; +import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerUtils; import org.apache.pulsar.functions.worker.service.api.Sinks; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; @@ -141,7 +143,7 @@ public void registerSink(final String tenant, // validate parameters try { if (isNotBlank(sinkPkgUrl)) { - componentPackageFile = getPackageFile(sinkPkgUrl); + componentPackageFile = getPackageFile(componentType, sinkPkgUrl); functionDetails = validateUpdateRequestParams(tenant, namespace, sinkName, sinkConfig, componentPackageFile); } else { @@ -308,6 +310,7 @@ public void updateSink(final String tenant, // validate parameters try { componentPackageFile = getPackageFile( + componentType, sinkPkgUrl, existingComponent.getPackageLocation().getPackagePath(), uploadedInputStream); @@ -419,7 +422,8 @@ private void setTransformFunctionPackageLocation(Function.FunctionMetaData.Build try { String builtin = functionDetails.getBuiltin(); if (isBlank(builtin)) { - functionPackageFile = getPackageFile(transformFunction); + functionPackageFile = + getPackageFile(Function.FunctionDetails.ComponentType.FUNCTION, transformFunction); } Function.PackageLocationMetaData.Builder functionPackageLocation = getFunctionPackageLocation(functionMetaDataBuilder.build(), @@ -704,7 +708,7 @@ private Function.FunctionDetails validateUpdateRequestParams(final String tenant sinkConfig.setName(sinkName); org.apache.pulsar.common.functions.Utils.inferMissingArguments(sinkConfig); - ClassLoader classLoader = null; + ValidatableFunctionPackage connectorFunctionPackage = null; // check if sink is builtin and extract classloader if (!StringUtils.isEmpty(sinkConfig.getArchive())) { String archive = sinkConfig.getArchive(); @@ -716,45 +720,63 @@ private Function.FunctionDetails validateUpdateRequestParams(final String tenant if (connector == null) { throw new IllegalArgumentException("Built-in sink is not available"); } - classLoader = connector.getClassLoader(); + connectorFunctionPackage = connector.getConnectorFunctionPackage(); } } - boolean shouldCloseClassLoader = false; + boolean shouldCloseFunctionPackage = false; + ValidatableFunctionPackage transformFunctionPackage = null; + boolean shouldCloseTransformFunctionPackage = false; try { // if sink is not builtin, attempt to extract classloader from package file if it exists - if (classLoader == null && sinkPackageFile != null) { - classLoader = getClassLoaderFromPackage(sinkConfig.getClassName(), - sinkPackageFile, worker().getWorkerConfig().getNarExtractionDirectory()); - shouldCloseClassLoader = true; + WorkerConfig workerConfig = worker().getWorkerConfig(); + if (connectorFunctionPackage == null && sinkPackageFile != null) { + connectorFunctionPackage = + new FunctionFilePackage(sinkPackageFile, workerConfig.getNarExtractionDirectory(), + workerConfig.getEnableClassloadingOfExternalFiles(), ConnectorDefinition.class); + shouldCloseFunctionPackage = true; } - if (classLoader == null) { + if (connectorFunctionPackage == null) { throw new IllegalArgumentException("Sink package is not provided"); } - ClassLoader functionClassLoader = null; if (isNotBlank(sinkConfig.getTransformFunction())) { - functionClassLoader = - getBuiltinFunctionClassLoader(sinkConfig.getTransformFunction()); - if (functionClassLoader == null) { - File functionPackageFile = getPackageFile(sinkConfig.getTransformFunction()); - functionClassLoader = getClassLoaderFromPackage(sinkConfig.getTransformFunctionClassName(), - functionPackageFile, worker().getWorkerConfig().getNarExtractionDirectory()); + transformFunctionPackage = + getBuiltinFunctionPackage(sinkConfig.getTransformFunction()); + if (transformFunctionPackage == null) { + File functionPackageFile = getPackageFile(Function.FunctionDetails.ComponentType.FUNCTION, + sinkConfig.getTransformFunction()); + transformFunctionPackage = + new FunctionFilePackage(functionPackageFile, workerConfig.getNarExtractionDirectory(), + workerConfig.getEnableClassloadingOfExternalFiles(), ConnectorDefinition.class); + shouldCloseTransformFunctionPackage = true; } - if (functionClassLoader == null) { + if (transformFunctionPackage == null) { throw new IllegalArgumentException("Transform Function package not found"); } } - SinkConfigUtils.ExtractedSinkDetails sinkDetails = SinkConfigUtils.validateAndExtractDetails( - sinkConfig, classLoader, functionClassLoader, worker().getWorkerConfig().getValidateConnectorConfig()); + SinkConfigUtils.ExtractedSinkDetails sinkDetails = + SinkConfigUtils.validateAndExtractDetails(sinkConfig, connectorFunctionPackage, + transformFunctionPackage, workerConfig.getValidateConnectorConfig()); return SinkConfigUtils.convert(sinkConfig, sinkDetails); } finally { - if (shouldCloseClassLoader) { - ClassLoaderUtils.closeClassLoader(classLoader); + if (shouldCloseFunctionPackage && connectorFunctionPackage instanceof AutoCloseable) { + try { + ((AutoCloseable) connectorFunctionPackage).close(); + } catch (Exception e) { + log.error("Failed to connector function file", e); + } + } + if (shouldCloseTransformFunctionPackage && transformFunctionPackage instanceof AutoCloseable) { + try { + ((AutoCloseable) transformFunctionPackage).close(); + } catch (Exception e) { + log.error("Failed to close transform function file", e); + } } } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java index c55ddf48b06b9..5191306146951 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java @@ -46,18 +46,20 @@ import org.apache.pulsar.common.io.SourceConfig; import org.apache.pulsar.common.policies.data.ExceptionInformation; import org.apache.pulsar.common.policies.data.SourceStatus; -import org.apache.pulsar.common.util.ClassLoaderUtils; import org.apache.pulsar.common.util.RestException; import org.apache.pulsar.functions.auth.FunctionAuthData; import org.apache.pulsar.functions.instance.InstanceUtils; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.utils.ComponentTypeUtils; +import org.apache.pulsar.functions.utils.FunctionFilePackage; import org.apache.pulsar.functions.utils.FunctionMetaDataUtils; import org.apache.pulsar.functions.utils.SourceConfigUtils; +import org.apache.pulsar.functions.utils.ValidatableFunctionPackage; import org.apache.pulsar.functions.utils.io.Connector; import org.apache.pulsar.functions.worker.FunctionMetaDataManager; import org.apache.pulsar.functions.worker.PulsarWorkerService; +import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerUtils; import org.apache.pulsar.functions.worker.service.api.Sources; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; @@ -141,7 +143,7 @@ public void registerSource(final String tenant, // validate parameters try { if (isPkgUrlProvided) { - componentPackageFile = getPackageFile(sourcePkgUrl); + componentPackageFile = getPackageFile(componentType, sourcePkgUrl); functionDetails = validateUpdateRequestParams(tenant, namespace, sourceName, sourceConfig, componentPackageFile); } else { @@ -302,6 +304,7 @@ public void updateSource(final String tenant, // validate parameters try { componentPackageFile = getPackageFile( + componentType, sourcePkgUrl, existingComponent.getPackageLocation().getPackagePath(), uploadedInputStream); @@ -663,7 +666,7 @@ private Function.FunctionDetails validateUpdateRequestParams(final String tenant sourceConfig.setName(sourceName); org.apache.pulsar.common.functions.Utils.inferMissingArguments(sourceConfig); - ClassLoader classLoader = null; + ValidatableFunctionPackage connectorFunctionPackage = null; // check if source is builtin and extract classloader if (!StringUtils.isEmpty(sourceConfig.getArchive())) { String archive = sourceConfig.getArchive(); @@ -675,30 +678,37 @@ private Function.FunctionDetails validateUpdateRequestParams(final String tenant if (connector == null) { throw new IllegalArgumentException("Built-in source is not available"); } - classLoader = connector.getClassLoader(); + connectorFunctionPackage = connector.getConnectorFunctionPackage(); } } - boolean shouldCloseClassLoader = false; + boolean shouldCloseFunctionPackage = false; try { // if source is not builtin, attempt to extract classloader from package file if it exists - if (classLoader == null && sourcePackageFile != null) { - classLoader = getClassLoaderFromPackage(sourceConfig.getClassName(), - sourcePackageFile, worker().getWorkerConfig().getNarExtractionDirectory()); - shouldCloseClassLoader = true; + WorkerConfig workerConfig = worker().getWorkerConfig(); + if (connectorFunctionPackage == null && sourcePackageFile != null) { + connectorFunctionPackage = + new FunctionFilePackage(sourcePackageFile, workerConfig.getNarExtractionDirectory(), + workerConfig.getEnableClassloadingOfExternalFiles(), ConnectorDefinition.class); + shouldCloseFunctionPackage = true; } - if (classLoader == null) { + if (connectorFunctionPackage == null) { throw new IllegalArgumentException("Source package is not provided"); } SourceConfigUtils.ExtractedSourceDetails sourceDetails = SourceConfigUtils.validateAndExtractDetails( - sourceConfig, classLoader, worker().getWorkerConfig().getValidateConnectorConfig()); + sourceConfig, connectorFunctionPackage, + workerConfig.getValidateConnectorConfig()); return SourceConfigUtils.convert(sourceConfig, sourceDetails); } finally { - if (shouldCloseClassLoader) { - ClassLoaderUtils.closeClassLoader(classLoader); + if (shouldCloseFunctionPackage && connectorFunctionPackage instanceof AutoCloseable) { + try { + ((AutoCloseable) connectorFunctionPackage).close(); + } catch (Exception e) { + log.error("Failed to connector function file", e); + } } } } diff --git a/pulsar-functions/worker/src/main/resources/findbugsExclude.xml b/pulsar-functions/worker/src/main/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-functions/worker/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/worker/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java index 4e4c3d2f234aa..ac5ca617ea43b 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java @@ -30,6 +30,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; import static org.testng.AssertJUnit.fail; +import java.util.List; import java.util.Map; import java.util.Optional; import org.apache.distributedlog.api.namespace.Namespace; @@ -77,7 +78,8 @@ public void testStartFunctionWithDLNamespace() throws Exception { @SuppressWarnings("resource") FunctionActioner actioner = new FunctionActioner(workerConfig, factory, dlogNamespace, - new ConnectorsManager(workerConfig), new FunctionsManager(workerConfig), mock(PulsarAdmin.class)); + new ConnectorsManager(workerConfig), new FunctionsManager(workerConfig), mock(PulsarAdmin.class), + mock(PackageUrlValidator.class)); Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder() .setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant") .setNamespace("test-namespace").setName("func-1")) @@ -109,6 +111,8 @@ public void testStartFunctionWithPkgUrl() throws Exception { workerConfig.setPulsarServiceUrl("pulsar://localhost:6650"); workerConfig.setStateStorageServiceUrl("foo"); workerConfig.setFunctionAssignmentTopicName("assignments"); + workerConfig.setAdditionalEnabledFunctionsUrlPatterns(List.of("file:///user/.*", "http://invalid/.*")); + workerConfig.setAdditionalEnabledConnectorUrlPatterns(List.of("file:///user/.*", "http://invalid/.*")); String downloadDir = this.getClass().getProtectionDomain().getCodeSource().getLocation().getPath(); workerConfig.setDownloadDirectory(downloadDir); @@ -122,11 +126,12 @@ public void testStartFunctionWithPkgUrl() throws Exception { @SuppressWarnings("resource") FunctionActioner actioner = new FunctionActioner(workerConfig, factory, dlogNamespace, - new ConnectorsManager(workerConfig), new FunctionsManager(workerConfig), mock(PulsarAdmin.class)); + new ConnectorsManager(workerConfig), new FunctionsManager(workerConfig), mock(PulsarAdmin.class), + new PackageUrlValidator(workerConfig)); // (1) test with file url. functionActioner should be able to consider file-url and it should be able to call // RuntimeSpawner - String pkgPathLocation = FILE + ":/user/my-file.jar"; + String pkgPathLocation = FILE + ":///user/my-file.jar"; startFunction(actioner, pkgPathLocation, pkgPathLocation); verify(runtime, times(1)).start(); @@ -194,7 +199,8 @@ public void testFunctionAuthDisabled() throws Exception { @SuppressWarnings("resource") FunctionActioner actioner = new FunctionActioner(workerConfig, factory, dlogNamespace, - new ConnectorsManager(workerConfig), new FunctionsManager(workerConfig), mock(PulsarAdmin.class)); + new ConnectorsManager(workerConfig), new FunctionsManager(workerConfig), mock(PulsarAdmin.class), + mock(PackageUrlValidator.class)); String pkgPathLocation = "http://invalid/my-file.jar"; @@ -257,7 +263,8 @@ public void testStartFunctionWithPackageUrl() throws Exception { @SuppressWarnings("resource") FunctionActioner actioner = new FunctionActioner(workerConfig, factory, dlogNamespace, - new ConnectorsManager(workerConfig), new FunctionsManager(workerConfig), pulsarAdmin); + new ConnectorsManager(workerConfig), new FunctionsManager(workerConfig), pulsarAdmin, + mock(PackageUrlValidator.class)); // (1) test with file url. functionActioner should be able to consider file-url and it should be able to call // RuntimeSpawner diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java index c332b5e646171..f2ea39651a26f 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java @@ -726,7 +726,7 @@ public void testExternallyManagedRuntimeUpdate() throws Exception { FunctionActioner functionActioner = spy(new FunctionActioner( workerConfig, - kubernetesRuntimeFactory, null, null, null, null)); + kubernetesRuntimeFactory, null, null, null, null, workerService.getPackageUrlValidator())); try (final MockedStatic runtimeFactoryMockedStatic = Mockito .mockStatic(RuntimeFactory.class);) { diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java index cfe087c78406a..73a229893e5d2 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java @@ -381,6 +381,6 @@ public static FunctionConfig createDefaultFunctionConfig() { public static Function.FunctionDetails createDefaultFunctionDetails() { FunctionConfig functionConfig = createDefaultFunctionConfig(); - return FunctionConfigUtils.convert(functionConfig, (ClassLoader) null); + return FunctionConfigUtils.convert(functionConfig); } } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java index 32a104c576993..a8415919c119b 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java @@ -18,1125 +18,82 @@ */ package org.apache.pulsar.functions.worker.rest.api.v2; - -import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; import static org.apache.pulsar.functions.utils.FunctionCommon.mergeJson; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; -import com.google.common.collect.Lists; import com.google.gson.Gson; -import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.LinkedList; import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.function.Consumer; import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; -import org.apache.distributedlog.api.namespace.Namespace; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.core.config.Configurator; import org.apache.pulsar.broker.authentication.AuthenticationParameters; -import org.apache.pulsar.client.admin.Functions; -import org.apache.pulsar.client.admin.Namespaces; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.admin.Tenants; import org.apache.pulsar.common.functions.FunctionConfig; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.functions.UpdateOptionsImpl; import org.apache.pulsar.common.util.RestException; -import org.apache.pulsar.functions.api.Context; -import org.apache.pulsar.functions.api.Function; -import org.apache.pulsar.functions.instance.InstanceUtils; -import org.apache.pulsar.functions.proto.Function.FunctionDetails; -import org.apache.pulsar.functions.proto.Function.FunctionMetaData; -import org.apache.pulsar.functions.proto.Function.PackageLocationMetaData; -import org.apache.pulsar.functions.proto.Function.ProcessingGuarantees; -import org.apache.pulsar.functions.proto.Function.SinkSpec; -import org.apache.pulsar.functions.proto.Function.SourceSpec; -import org.apache.pulsar.functions.proto.Function.SubscriptionType; -import org.apache.pulsar.functions.runtime.RuntimeFactory; -import org.apache.pulsar.functions.source.TopicSchema; -import org.apache.pulsar.functions.utils.FunctionCommon; +import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.utils.FunctionConfigUtils; -import org.apache.pulsar.functions.worker.FunctionMetaDataManager; -import org.apache.pulsar.functions.worker.FunctionRuntimeManager; -import org.apache.pulsar.functions.worker.LeaderService; -import org.apache.pulsar.functions.worker.PulsarWorkerService; -import org.apache.pulsar.functions.worker.WorkerConfig; -import org.apache.pulsar.functions.worker.WorkerUtils; -import org.apache.pulsar.functions.worker.rest.api.FunctionsImpl; import org.apache.pulsar.functions.worker.rest.api.FunctionsImplV2; -import org.apache.pulsar.functions.worker.rest.api.PulsarFunctionTestTemporaryDirectory; +import org.apache.pulsar.functions.worker.rest.api.v3.AbstractFunctionApiResourceTest; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.testng.Assert; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -/** - * Unit test of {@link FunctionsApiV2Resource}. - */ -public class FunctionApiV2ResourceTest { - - private static final class TestFunction implements Function { - - @Override - public String process(String input, Context context) { - return input; - } - } - - private static final String tenant = "test-tenant"; - private static final String namespace = "test-namespace"; - private static final String function = "test-function"; - private static final String outputTopic = "test-output-topic"; - private static final String outputSerdeClassName = TopicSchema.DEFAULT_SERDE; - private static final String className = TestFunction.class.getName(); - private SubscriptionType subscriptionType = SubscriptionType.FAILOVER; - private static final Map topicsToSerDeClassName = new HashMap<>(); - static { - topicsToSerDeClassName.put("persistent://public/default/test_src", TopicSchema.DEFAULT_SERDE); - } - private static final int parallelism = 1; - - private PulsarWorkerService mockedWorkerService; - private PulsarAdmin mockedPulsarAdmin; - private Tenants mockedTenants; - private Namespaces mockedNamespaces; - private Functions mockedFunctions; - private TenantInfoImpl mockedTenantInfo; - private List namespaceList = new LinkedList<>(); - private FunctionMetaDataManager mockedManager; - private FunctionRuntimeManager mockedFunctionRunTimeManager; - private RuntimeFactory mockedRuntimeFactory; - private Namespace mockedNamespace; +public class FunctionApiV2ResourceTest extends AbstractFunctionApiResourceTest { private FunctionsImplV2 resource; - private InputStream mockedInputStream; - private FormDataContentDisposition mockedFormData; - private FunctionMetaData mockedFunctionMetadata; - private LeaderService mockedLeaderService; - private PulsarFunctionTestTemporaryDirectory tempDirectory; - private static Map mockStaticContexts = new HashMap<>(); - - @BeforeMethod - public void setup() throws Exception { - this.mockedManager = mock(FunctionMetaDataManager.class); - this.mockedFunctionRunTimeManager = mock(FunctionRuntimeManager.class); - this.mockedTenantInfo = mock(TenantInfoImpl.class); - this.mockedRuntimeFactory = mock(RuntimeFactory.class); - this.mockedInputStream = mock(InputStream.class); - this.mockedNamespace = mock(Namespace.class); - this.mockedFormData = mock(FormDataContentDisposition.class); - when(mockedFormData.getFileName()).thenReturn("test"); - this.mockedPulsarAdmin = mock(PulsarAdmin.class); - this.mockedTenants = mock(Tenants.class); - this.mockedNamespaces = mock(Namespaces.class); - this.mockedFunctions = mock(Functions.class); - this.mockedLeaderService = mock(LeaderService.class); - this.mockedFunctionMetadata = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); - namespaceList.add(tenant + "/" + namespace); - - this.mockedWorkerService = mock(PulsarWorkerService.class); - when(mockedWorkerService.getFunctionMetaDataManager()).thenReturn(mockedManager); - when(mockedWorkerService.getLeaderService()).thenReturn(mockedLeaderService); - when(mockedWorkerService.getFunctionRuntimeManager()).thenReturn(mockedFunctionRunTimeManager); - when(mockedFunctionRunTimeManager.getRuntimeFactory()).thenReturn(mockedRuntimeFactory); - when(mockedWorkerService.getDlogNamespace()).thenReturn(mockedNamespace); - when(mockedWorkerService.isInitialized()).thenReturn(true); - when(mockedWorkerService.getBrokerAdmin()).thenReturn(mockedPulsarAdmin); - when(mockedWorkerService.getFunctionAdmin()).thenReturn(mockedPulsarAdmin); - when(mockedPulsarAdmin.tenants()).thenReturn(mockedTenants); - when(mockedPulsarAdmin.namespaces()).thenReturn(mockedNamespaces); - when(mockedPulsarAdmin.functions()).thenReturn(mockedFunctions); - when(mockedTenants.getTenantInfo(any())).thenReturn(mockedTenantInfo); - when(mockedNamespaces.getNamespaces(any())).thenReturn(namespaceList); - when(mockedLeaderService.isLeader()).thenReturn(true); - when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetadata); - - // worker config - WorkerConfig workerConfig = new WorkerConfig() - .setWorkerId("test") - .setWorkerPort(8080) - .setFunctionMetadataTopicName("pulsar/functions") - .setNumFunctionPackageReplicas(3) - .setPulsarServiceUrl("pulsar://localhost:6650/"); - tempDirectory = PulsarFunctionTestTemporaryDirectory.create(getClass().getSimpleName()); - tempDirectory.useTemporaryDirectoriesForWorkerConfig(workerConfig); - when(mockedWorkerService.getWorkerConfig()).thenReturn(workerConfig); - - FunctionsImpl functions = spy(new FunctionsImpl(() -> mockedWorkerService)); - - this.resource = spy(new FunctionsImplV2(functions)); - - } - - @AfterMethod(alwaysRun = true) - public void cleanup() { - if (tempDirectory != null) { - tempDirectory.delete(); - } - mockStaticContexts.values().forEach(MockedStatic::close); - mockStaticContexts.clear(); - } - - private void mockStatic(Class classStatic, Consumer> consumer) { - final MockedStatic mockedStatic = - mockStaticContexts.computeIfAbsent(classStatic.getName(), name -> Mockito.mockStatic(classStatic)); - consumer.accept(mockedStatic); - } - - private void mockWorkerUtils() { - mockStatic(WorkerUtils.class, ctx -> { - ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); - }); - } - - private void mockWorkerUtils(Consumer> consumer) { - mockStatic(WorkerUtils.class, ctx -> { - ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); - if (consumer != null) { - consumer.accept(ctx); - } - }); - } - - private void mockInstanceUtils() { - mockStatic(InstanceUtils.class, ctx -> { - ctx.when(() -> InstanceUtils.calculateSubjectType(any())) - .thenReturn(FunctionDetails.ComponentType.FUNCTION); - }); - } - - // - // Register Functions - // - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") - public void testRegisterFunctionMissingTenant() { - try { - testRegisterFunctionMissingArguments( - null, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") - public void testRegisterFunctionMissingNamespace() { - try { - testRegisterFunctionMissingArguments( - tenant, - null, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") - public void testRegisterFunctionMissingFunctionName() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - null, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function package is not provided") - public void testRegisterFunctionMissingPackage() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - null, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "No input topic\\(s\\) specified for the function") - public void testRegisterFunctionMissingInputTopics() throws Exception { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - null, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function Package is not provided") - public void testRegisterFunctionMissingPackageDetails() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - null, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function package does not have" - + " the correct format. Pulsar cannot determine if the package is a NAR package or JAR package. Function " - + "classname is not provided and attempts to load it as a NAR package produced the following error.*") - public void testRegisterFunctionMissingClassName() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - null, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function class UnknownClass must be in class path") - public void testRegisterFunctionWrongClassName() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - "UnknownClass", - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function parallelism must be a positive number") - public void testRegisterFunctionWrongParallelism() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - -2, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, - expectedExceptionsMessageRegExp = "Output topic persistent://public/default/test_src is also being used as an input topic \\(topics must be one or the other\\)") - public void testRegisterFunctionSameInputOutput() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - topicsToSerDeClassName.keySet().iterator().next(), - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Output topic " + function + - "-output-topic/test:" + " is invalid") - public void testRegisterFunctionWrongOutputTopic() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - function + "-output-topic/test:", - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Encountered error .*. when getting Function package from .*") - public void testRegisterFunctionHttpUrl() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - null, - topicsToSerDeClassName, - null, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "http://localhost:1234/test"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - private void testRegisterFunctionMissingArguments( - String tenant, - String namespace, - String function, - InputStream inputStream, - Map topicsToSerDeClassName, - FormDataContentDisposition details, - String outputTopic, - String outputSerdeClassName, - String className, - Integer parallelism, - String functionPkgUrl) { - FunctionConfig functionConfig = new FunctionConfig(); - if (tenant != null) { - functionConfig.setTenant(tenant); - } - if (namespace != null) { - functionConfig.setNamespace(namespace); - } - if (function != null) { - functionConfig.setName(function); - } - if (topicsToSerDeClassName != null) { - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - } - if (outputTopic != null) { - functionConfig.setOutput(outputTopic); - } - if (outputSerdeClassName != null) { - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - } - if (className != null) { - functionConfig.setClassName(className); - } - if (parallelism != null) { - functionConfig.setParallelism(parallelism); - } - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - - mockWorkerUtils(ctx -> { - ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class))) - .thenCallRealMethod(); - }); - - try { - resource.registerFunction( - tenant, - namespace, - function, - inputStream, - details, - functionPkgUrl, - JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), - AuthenticationParameters.builder().build()); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - - } - - private void registerDefaultFunction() { - FunctionConfig functionConfig = createDefaultFunctionConfig(); - try { - resource.registerFunction( - tenant, - namespace, - function, - mockedInputStream, - mockedFormData, - null, - JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), - AuthenticationParameters.builder().build()); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function already exists") - public void testRegisterExistedFunction() { - try { - Configurator.setRootLevel(Level.DEBUG); - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure") - public void testRegisterFunctionUploadFailure() throws Exception { - try { - mockWorkerUtils(ctx -> { - ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class))) - .thenThrow(new IOException("upload failure")); - }); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); - throw re; - } - } - - @Test - public void testRegisterFunctionSuccess() throws Exception { - try { - try (MockedStatic mocked = Mockito.mockStatic(WorkerUtils.class)) { - mocked.when(() -> WorkerUtils.uploadToBookKeeper( - any(Namespace.class), - any(InputStream.class), - anyString())).thenAnswer((i) -> null); - mocked.when(() -> WorkerUtils.dumpToTmpFile(any())).thenAnswer(i -> - { - try { - File tmpFile = FunctionCommon.createPkgTempFile(); - tmpFile.deleteOnExit(); - Files.copy((InputStream) i.getArguments()[0], tmpFile.toPath(), REPLACE_EXISTING); - return tmpFile; - } catch (IOException e) { - throw new RuntimeException("Cannot create a temporary file", e); - } - - } - ); - WorkerUtils.uploadToBookKeeper(null, null, null); - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - - registerDefaultFunction(); - } - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace does not exist") - public void testRegisterFunctionNonExistingNamespace() { - try { - this.namespaceList.clear(); - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant does not exist") - public void testRegisterFunctionNonexistantTenant() throws Exception { - try { - when(mockedTenants.getTenantInfo(any())).thenThrow(PulsarAdminException.NotFoundException.class); - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to register") - public void testRegisterFunctionFailure() throws Exception { - try { - mockWorkerUtils(); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - doThrow(new IllegalArgumentException("function failed to register")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean()); - - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registration interrupted") - public void testRegisterFunctionInterrupted() throws Exception { - try { - mockWorkerUtils(); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - - doThrow(new IllegalStateException("Function registration interrupted")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean()); - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); - throw re; - } - } - - // - // Update Functions - // - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") - public void testUpdateFunctionMissingTenant() throws Exception { - try { - testUpdateFunctionMissingArguments( - null, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "Tenant is not provided"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") - public void testUpdateFunctionMissingNamespace() throws Exception { - try { - testUpdateFunctionMissingArguments( - tenant, - null, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "Namespace is not provided"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") - public void testUpdateFunctionMissingFunctionName() throws Exception { - try { - testUpdateFunctionMissingArguments( - tenant, - namespace, - null, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "Function name is not provided"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") - public void testUpdateFunctionMissingPackage() throws Exception { - try { - mockWorkerUtils(); - testUpdateFunctionMissingArguments( - tenant, - namespace, - function, - null, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "Update contains no change"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") - public void testUpdateFunctionMissingInputTopic() throws Exception { - try { - mockWorkerUtils(); - - testUpdateFunctionMissingArguments( - tenant, - namespace, - function, - null, - null, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "Update contains no change"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") - public void testUpdateFunctionMissingClassName() throws Exception { - try { - mockWorkerUtils(); - - testUpdateFunctionMissingArguments( - tenant, - namespace, - function, - null, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - null, - parallelism, - "Update contains no change"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test - public void testUpdateFunctionChangedParallelism() throws Exception { - try { - mockWorkerUtils(); - - testUpdateFunctionMissingArguments( - tenant, - namespace, - function, - null, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - null, - parallelism + 1, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } + @Override + protected void doSetup() { + super.doSetup(); + this.resource = spy(new FunctionsImplV2(() -> mockedWorkerService)); } - @Test - public void testUpdateFunctionChangedInputs() throws Exception { - mockWorkerUtils(); - - testUpdateFunctionMissingArguments( + protected void registerFunction(String tenant, String namespace, String function, InputStream inputStream, + FormDataContentDisposition details, String functionPkgUrl, + FunctionConfig functionConfig) throws IOException { + resource.registerFunction( tenant, namespace, function, - null, - topicsToSerDeClassName, - mockedFormData, - "DifferentOutput", - outputSerdeClassName, - null, - parallelism, + inputStream, + details, + functionPkgUrl, + JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig)), null); } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Input Topics cannot be altered") - public void testUpdateFunctionChangedOutput() throws Exception { - try { - mockWorkerUtils(); - - Map someOtherInput = new HashMap<>(); - someOtherInput.put("DifferentTopic", TopicSchema.DEFAULT_SERDE); - testUpdateFunctionMissingArguments( - tenant, - namespace, - function, - null, - someOtherInput, - mockedFormData, - outputTopic, - outputSerdeClassName, - null, - parallelism, - "Input Topics cannot be altered"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - private void testUpdateFunctionMissingArguments( - String tenant, - String namespace, - String function, - InputStream inputStream, - Map topicsToSerDeClassName, - FormDataContentDisposition details, - String outputTopic, - String outputSerdeClassName, - String className, - Integer parallelism, - String expectedError) throws Exception { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - FunctionConfig functionConfig = new FunctionConfig(); - if (tenant != null) { - functionConfig.setTenant(tenant); - } - if (namespace != null) { - functionConfig.setNamespace(namespace); - } - if (function != null) { - functionConfig.setName(function); - } - if (topicsToSerDeClassName != null) { - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - } - if (outputTopic != null) { - functionConfig.setOutput(outputTopic); - } - if (outputSerdeClassName != null) { - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - } - if (className != null) { - functionConfig.setClassName(className); - } - if (parallelism != null) { - functionConfig.setParallelism(parallelism); - } - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - - if (expectedError != null) { - doThrow(new IllegalArgumentException(expectedError)) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean()); - } - - try { - resource.updateFunction( - tenant, - namespace, - function, - inputStream, - details, - null, - JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), - AuthenticationParameters.builder().build()); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - - } - - private void updateDefaultFunction() { - FunctionConfig functionConfig = new FunctionConfig(); - functionConfig.setTenant(tenant); - functionConfig.setNamespace(namespace); - functionConfig.setName(function); - functionConfig.setClassName(className); - functionConfig.setParallelism(parallelism); - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - functionConfig.setOutput(outputTopic); - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - - try { - resource.updateFunction( - tenant, - namespace, - function, - mockedInputStream, - mockedFormData, - null, - JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), - AuthenticationParameters.builder().build()); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist") - public void testUpdateNotExistedFunction() { - try { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - updateDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure") - public void testUpdateFunctionUploadFailure() throws Exception { - try { - mockWorkerUtils(ctx -> { - ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class))) - .thenThrow(new IOException("upload failure")); - }); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - updateDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); - throw re; - } - } - - @Test - public void testUpdateFunctionSuccess() throws Exception { - mockWorkerUtils(); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - updateDefaultFunction(); - } - - @Test - public void testUpdateFunctionWithUrl() throws Exception { - Configurator.setRootLevel(Level.DEBUG); - - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath().replace('\\', '/'); - String filePackageUrl = "file:///" + fileLocation; - - FunctionConfig functionConfig = new FunctionConfig(); - functionConfig.setOutput(outputTopic); - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - functionConfig.setTenant(tenant); - functionConfig.setNamespace(namespace); - functionConfig.setName(function); - functionConfig.setClassName(className); - functionConfig.setParallelism(parallelism); - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - try { - resource.updateFunction( - tenant, - namespace, - function, - null, - null, - filePackageUrl, - JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), - AuthenticationParameters.builder().build()); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to register") - public void testUpdateFunctionFailure() throws Exception { - try { - mockWorkerUtils(); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - doThrow(new IllegalArgumentException("function failed to register")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean()); - - updateDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registeration interrupted") - public void testUpdateFunctionInterrupted() throws Exception { - try { - mockWorkerUtils(); - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - doThrow(new IllegalStateException("Function registeration interrupted")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean()); - - updateDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); - throw re; - } - } - - // - // deregister function - // - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") - public void testDeregisterFunctionMissingTenant() { - try { - - testDeregisterFunctionMissingArguments( - null, - namespace, - function - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } + protected void updateFunction(String tenant, + String namespace, + String functionName, + InputStream uploadedInputStream, + FormDataContentDisposition fileDetail, + String functionPkgUrl, + FunctionConfig functionConfig, + AuthenticationParameters authParams, + UpdateOptionsImpl updateOptions) throws IOException { + resource.updateFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail, functionPkgUrl, + JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig)), authParams); } - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") - public void testDeregisterFunctionMissingNamespace() { - try { - testDeregisterFunctionMissingArguments( - tenant, - null, - function - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; + protected File downloadFunction(final String path, final AuthenticationParameters authParams) + throws IOException { + Response response = resource.downloadFunction(path, authParams); + StreamingOutput streamingOutput = readEntity(response, StreamingOutput.class); + File pkgFile = File.createTempFile("testpkg", "nar"); + try (OutputStream output = new FileOutputStream(pkgFile)) { + streamingOutput.write(output); } + return pkgFile; } - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") - public void testDeregisterFunctionMissingFunctionName() { - try { - testDeregisterFunctionMissingArguments( - tenant, - namespace, - null - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } + private T readEntity(Response response, Class clazz) { + return clazz.cast(response.getEntity()); } - private void testDeregisterFunctionMissingArguments( + protected void testDeregisterFunctionMissingArguments( String tenant, String namespace, String function @@ -1145,112 +102,18 @@ private void testDeregisterFunctionMissingArguments( tenant, namespace, function, - AuthenticationParameters.builder().build()); + null); } - private void deregisterDefaultFunction() { + protected void deregisterDefaultFunction() { resource.deregisterFunction( tenant, namespace, function, - AuthenticationParameters.builder().build()); - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist") - public void testDeregisterNotExistedFunction() { - try { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - deregisterDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.NOT_FOUND); - throw re; - } - } - - @Test - public void testDeregisterFunctionSuccess() { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - deregisterDefaultFunction(); - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to deregister") - public void testDeregisterFunctionFailure() throws Exception { - try { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - doThrow(new IllegalArgumentException("function failed to deregister")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean()); - - deregisterDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function deregisteration interrupted") - public void testDeregisterFunctionInterrupted() throws Exception { - try { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - doThrow(new IllegalStateException("Function deregisteration interrupted")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean()); - - deregisterDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); - throw re; - } - } - - // - // Get Function Info - // - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") - public void testGetFunctionMissingTenant() throws IOException { - try { - testGetFunctionMissingArguments( - null, - namespace, - function - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") - public void testGetFunctionMissingNamespace() throws IOException { - try { - testGetFunctionMissingArguments( - tenant, - null, - function - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") - public void testGetFunctionMissingFunctionName() throws IOException { - try { - testGetFunctionMissingArguments( - tenant, - namespace, - null - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } + null); } - private void testGetFunctionMissingArguments( + protected void testGetFunctionMissingArguments( String tenant, String namespace, String function @@ -1258,20 +121,36 @@ private void testGetFunctionMissingArguments( resource.getFunctionInfo( tenant, namespace, - function, - AuthenticationParameters.builder().build() + function, null + ); + } + + protected void testListFunctionsMissingArguments( + String tenant, + String namespace + ) { + resource.listFunctions( + tenant, + namespace, null ); } - private FunctionDetails getDefaultFunctionInfo() throws IOException { + protected List listDefaultFunctions() { + return new Gson().fromJson(readEntity(resource.listFunctions( + tenant, + namespace, null + ), String.class), List.class); + } + + private Function.FunctionDetails getDefaultFunctionInfo() throws IOException { String json = (String) resource.getFunctionInfo( tenant, namespace, function, AuthenticationParameters.builder().build() ).getEntity(); - FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder(); + Function.FunctionDetails.Builder functionDetailsBuilder = Function.FunctionDetails.newBuilder(); mergeJson(json, functionDetailsBuilder); return functionDetailsBuilder.build(); } @@ -1292,218 +171,31 @@ public void testGetFunctionSuccess() throws IOException { mockInstanceUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - SinkSpec sinkSpec = SinkSpec.newBuilder() + Function.SinkSpec sinkSpec = Function.SinkSpec.newBuilder() .setTopic(outputTopic) .setSerDeClassName(outputSerdeClassName).build(); - FunctionDetails functionDetails = FunctionDetails.newBuilder() + Function.FunctionDetails functionDetails = Function.FunctionDetails.newBuilder() .setClassName(className) .setSink(sinkSpec) .setName(function) .setNamespace(namespace) - .setProcessingGuarantees(ProcessingGuarantees.ATMOST_ONCE) + .setProcessingGuarantees(Function.ProcessingGuarantees.ATMOST_ONCE) .setAutoAck(true) .setTenant(tenant) .setParallelism(parallelism) - .setSource(SourceSpec.newBuilder().setSubscriptionType(subscriptionType) + .setSource(Function.SourceSpec.newBuilder().setSubscriptionType(subscriptionType) .putAllTopicsToSerDeClassName(topicsToSerDeClassName)).build(); - FunctionMetaData metaData = FunctionMetaData.newBuilder() + Function.FunctionMetaData metaData = Function.FunctionMetaData.newBuilder() .setCreateTime(System.currentTimeMillis()) .setFunctionDetails(functionDetails) - .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("/path/to/package")) + .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("/path/to/package")) .setVersion(1234) .build(); when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(function))).thenReturn(metaData); - FunctionDetails actual = getDefaultFunctionInfo(); + Function.FunctionDetails actual = getDefaultFunctionInfo(); assertEquals( functionDetails, actual); } - - // - // List Functions - // - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") - public void testListFunctionsMissingTenant() { - try { - testListFunctionsMissingArguments( - null, - namespace - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") - public void testListFunctionsMissingNamespace() { - try { - testListFunctionsMissingArguments( - tenant, - null - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - private void testListFunctionsMissingArguments( - String tenant, - String namespace - ) { - resource.listFunctions( - tenant, - namespace, - AuthenticationParameters.builder().build() - ); - - } - - private List listDefaultFunctions() { - return new Gson().fromJson((String) resource.listFunctions( - tenant, - namespace, - AuthenticationParameters.builder().build() - ).getEntity(), List.class); - } - - @Test - public void testListFunctionsSuccess() { - mockInstanceUtils(); - final List functions = Lists.newArrayList("test-1", "test-2"); - final List metaDataList = new LinkedList<>(); - FunctionMetaData functionMetaData1 = FunctionMetaData.newBuilder().setFunctionDetails( - FunctionDetails.newBuilder().setName("test-1").build() - ).build(); - FunctionMetaData functionMetaData2 = FunctionMetaData.newBuilder().setFunctionDetails( - FunctionDetails.newBuilder().setName("test-2").build() - ).build(); - metaDataList.add(functionMetaData1); - metaDataList.add(functionMetaData2); - when(mockedManager.listFunctions(eq(tenant), eq(namespace))).thenReturn(metaDataList); - - List functionList = listDefaultFunctions(); - assertEquals(functions, functionList); - } - - @Test - public void testDownloadFunctionHttpUrl() throws Exception { - String jarHttpUrl = - "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar"; - String testDir = FunctionApiV2ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); - FunctionsImplV2 function = new FunctionsImplV2(() -> mockedWorkerService); - StreamingOutput streamOutput = (StreamingOutput) function.downloadFunction(jarHttpUrl, - AuthenticationParameters.builder().build()).getEntity(); - File pkgFile = new File(testDir, UUID.randomUUID().toString()); - OutputStream output = new FileOutputStream(pkgFile); - streamOutput.write(output); - Assert.assertTrue(pkgFile.exists()); - if (pkgFile.exists()) { - pkgFile.delete(); - } - } - - @Test - public void testDownloadFunctionFile() throws Exception { - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath().replace('\\', '/'); - String testDir = FunctionApiV2ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); - FunctionsImplV2 function = new FunctionsImplV2(() -> mockedWorkerService); - StreamingOutput streamOutput = (StreamingOutput) function.downloadFunction("file:///" + fileLocation, - AuthenticationParameters.builder().build()).getEntity(); - File pkgFile = new File(testDir, UUID.randomUUID().toString()); - OutputStream output = new FileOutputStream(pkgFile); - streamOutput.write(output); - Assert.assertTrue(pkgFile.exists()); - if (pkgFile.exists()) { - pkgFile.delete(); - } - } - - @Test - public void testRegisterFunctionFileUrlWithValidSinkClass() throws Exception { - Configurator.setRootLevel(Level.DEBUG); - - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath().replace('\\', '/'); - String filePackageUrl = "file:///" + fileLocation; - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - - FunctionConfig functionConfig = new FunctionConfig(); - functionConfig.setTenant(tenant); - functionConfig.setNamespace(namespace); - functionConfig.setName(function); - functionConfig.setClassName(className); - functionConfig.setParallelism(parallelism); - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - functionConfig.setOutput(outputTopic); - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - try { - resource.registerFunction(tenant, namespace, function, null, null, filePackageUrl, - JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), - AuthenticationParameters.builder().build()); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - - } - - @Test - public void testRegisterFunctionWithConflictingFields() throws Exception { - Configurator.setRootLevel(Level.DEBUG); - String actualTenant = "DIFFERENT_TENANT"; - String actualNamespace = "DIFFERENT_NAMESPACE"; - String actualName = "DIFFERENT_NAME"; - this.namespaceList.add(actualTenant + "/" + actualNamespace); - - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath().replace('\\', '/'); - String filePackageUrl = "file:///" + fileLocation; - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - when(mockedManager.containsFunction(eq(actualTenant), eq(actualNamespace), eq(actualName))).thenReturn(false); - - FunctionConfig functionConfig = new FunctionConfig(); - functionConfig.setTenant(tenant); - functionConfig.setNamespace(namespace); - functionConfig.setName(function); - functionConfig.setClassName(className); - functionConfig.setParallelism(parallelism); - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - functionConfig.setOutput(outputTopic); - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - try { - resource.registerFunction(actualTenant, actualNamespace, actualName, null, null, filePackageUrl, - JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), - AuthenticationParameters.builder().build()); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - } - - public static FunctionConfig createDefaultFunctionConfig() { - FunctionConfig functionConfig = new FunctionConfig(); - functionConfig.setTenant(tenant); - functionConfig.setNamespace(namespace); - functionConfig.setName(function); - functionConfig.setClassName(className); - functionConfig.setParallelism(parallelism); - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - functionConfig.setOutput(outputTopic); - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - return functionConfig; - } - - public static FunctionDetails createDefaultFunctionDetails() { - FunctionConfig functionConfig = createDefaultFunctionConfig(); - return FunctionConfigUtils.convert(functionConfig, (ClassLoader) null); - } } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionApiResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionApiResourceTest.java new file mode 100644 index 0000000000000..388331ce6f241 --- /dev/null +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionApiResourceTest.java @@ -0,0 +1,1374 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.functions.worker.rest.api.v3; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import com.google.common.collect.Lists; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import javax.ws.rs.core.Response; +import org.apache.distributedlog.api.namespace.Namespace; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.config.Configurator; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.common.functions.FunctionConfig; +import org.apache.pulsar.common.functions.UpdateOptionsImpl; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.RestException; +import org.apache.pulsar.functions.api.Context; +import org.apache.pulsar.functions.api.Function; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; +import org.apache.pulsar.functions.proto.Function.FunctionMetaData; +import org.apache.pulsar.functions.proto.Function.SubscriptionType; +import org.apache.pulsar.functions.source.TopicSchema; +import org.apache.pulsar.functions.utils.FunctionConfigUtils; +import org.apache.pulsar.functions.worker.WorkerConfig; +import org.apache.pulsar.functions.worker.WorkerUtils; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.Test; + +public abstract class AbstractFunctionApiResourceTest extends AbstractFunctionsResourceTest { + @Override + protected void customizeWorkerConfig(WorkerConfig workerConfig, Method method) { + if (method.getName().contains("Upload")) { + workerConfig.setFunctionsWorkerEnablePackageManagement(false); + } + } + + @Test + public void testListFunctionsSuccess() { + mockInstanceUtils(); + final List functions = Lists.newArrayList("test-1", "test-2"); + final List metaDataList = new LinkedList<>(); + FunctionMetaData functionMetaData1 = FunctionMetaData.newBuilder().setFunctionDetails( + FunctionDetails.newBuilder().setName("test-1").build() + ).build(); + FunctionMetaData functionMetaData2 = FunctionMetaData.newBuilder().setFunctionDetails( + FunctionDetails.newBuilder().setName("test-2").build() + ).build(); + metaDataList.add(functionMetaData1); + metaDataList.add(functionMetaData2); + when(mockedManager.listFunctions(eq(tenant), eq(namespace))).thenReturn(metaDataList); + + List functionList = listDefaultFunctions(); + assertEquals(functions, functionList); + } + + @Test + public void testOnlyGetSources() { + List functions = Lists.newArrayList("test-2"); + List functionMetaDataList = new LinkedList<>(); + FunctionMetaData f1 = FunctionMetaData.newBuilder().setFunctionDetails( + FunctionDetails.newBuilder() + .setName("test-1") + .setComponentType(FunctionDetails.ComponentType.SOURCE) + .build()).build(); + functionMetaDataList.add(f1); + FunctionMetaData f2 = FunctionMetaData.newBuilder().setFunctionDetails( + FunctionDetails.newBuilder() + .setName("test-2") + .setComponentType(FunctionDetails.ComponentType.FUNCTION) + .build()).build(); + functionMetaDataList.add(f2); + FunctionMetaData f3 = FunctionMetaData.newBuilder().setFunctionDetails( + FunctionDetails.newBuilder() + .setName("test-3") + .setComponentType(FunctionDetails.ComponentType.SINK) + .build()).build(); + functionMetaDataList.add(f3); + when(mockedManager.listFunctions(eq(tenant), eq(namespace))).thenReturn(functionMetaDataList); + + List functionList = listDefaultFunctions(); + assertEquals(functions, functionList); + } + + private static final class TestFunction implements Function { + + @Override + public String process(String input, Context context) { + return input; + } + } + + private static final class WrongFunction implements Consumer { + @Override + public void accept(String s) { + + } + } + + protected static final String function = "test-function"; + protected static final String outputTopic = "test-output-topic"; + protected static final String outputSerdeClassName = TopicSchema.DEFAULT_SERDE; + protected static final String className = TestFunction.class.getName(); + protected SubscriptionType subscriptionType = SubscriptionType.FAILOVER; + protected FunctionMetaData mockedFunctionMetadata; + + + @Override + protected void doSetup() { + this.mockedFunctionMetadata = + FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); + when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetadata); + } + + @Override + protected FunctionDetails.ComponentType getComponentType() { + return FunctionDetails.ComponentType.FUNCTION; + } + + + abstract protected void registerFunction(String tenant, String namespace, String function, InputStream inputStream, + FormDataContentDisposition details, String functionPkgUrl, FunctionConfig functionConfig) + throws IOException; + abstract protected void updateFunction(String tenant, + String namespace, + String functionName, + InputStream uploadedInputStream, + FormDataContentDisposition fileDetail, + String functionPkgUrl, + FunctionConfig functionConfig, + AuthenticationParameters authParams, + UpdateOptionsImpl updateOptions) throws IOException; + + abstract protected File downloadFunction(final String path, final AuthenticationParameters authParams) + throws IOException; + + abstract protected void testDeregisterFunctionMissingArguments( + String tenant, + String namespace, + String function + ); + + abstract protected void deregisterDefaultFunction(); + + abstract protected void testGetFunctionMissingArguments( + String tenant, + String namespace, + String function + ) throws IOException; + + abstract protected void testListFunctionsMissingArguments( + String tenant, + String namespace + ); + + abstract protected List listDefaultFunctions(); + + // + // Register Functions + // + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") + public void testRegisterFunctionMissingTenant() throws IOException { + try { + testRegisterFunctionMissingArguments( + null, + namespace, + function, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + className, + parallelism, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") + public void testRegisterFunctionMissingNamespace() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + null, + function, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + className, + parallelism, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") + public void testRegisterFunctionMissingFunctionName() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + namespace, + null, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + className, + parallelism, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function package is not " + + "provided") + public void testRegisterFunctionMissingPackage() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + namespace, + function, + null, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + className, + parallelism, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "No input topic\\(s\\) " + + "specified for the function") + public void testRegisterFunctionMissingInputTopics() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + namespace, + function, + mockedInputStream, + null, + mockedFormData, + outputTopic, + outputSerdeClassName, + className, + parallelism, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function Package is not " + + "provided") + public void testRegisterFunctionMissingPackageDetails() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + namespace, + function, + mockedInputStream, + topicsToSerDeClassName, + null, + outputTopic, + outputSerdeClassName, + className, + parallelism, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, + expectedExceptionsMessageRegExp = "Function class name is not provided.") + public void testRegisterFunctionMissingClassName() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + namespace, + function, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + null, + parallelism, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function class UnknownClass " + + "must be in class path") + public void testRegisterFunctionWrongClassName() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + namespace, + function, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + "UnknownClass", + parallelism, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function parallelism must be a" + + " positive number") + public void testRegisterFunctionWrongParallelism() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + namespace, + function, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + className, + -2, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, + expectedExceptionsMessageRegExp = "Output topic persistent://public/default/test_src is also being used " + + "as an input topic \\(topics must be one or the other\\)") + public void testRegisterFunctionSameInputOutput() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + namespace, + function, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + topicsToSerDeClassName.keySet().iterator().next(), + outputSerdeClassName, + className, + parallelism, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Output topic " + function + + "-output-topic/test:" + " is invalid") + public void testRegisterFunctionWrongOutputTopic() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + namespace, + function, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + function + "-output-topic/test:", + outputSerdeClassName, + className, + parallelism, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Encountered error .*. when " + + "getting Function package from .*") + public void testRegisterFunctionHttpUrl() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + namespace, + function, + null, + topicsToSerDeClassName, + null, + outputTopic, + outputSerdeClassName, + className, + parallelism, + "http://localhost:1234/test"); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function class .*. does not " + + "implement the correct interface") + public void testRegisterFunctionImplementWrongInterface() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + namespace, + function, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + WrongFunction.class.getName(), + parallelism, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + private void testRegisterFunctionMissingArguments( + String tenant, + String namespace, + String function, + InputStream inputStream, + Map topicsToSerDeClassName, + FormDataContentDisposition details, + String outputTopic, + String outputSerdeClassName, + String className, + Integer parallelism, + String functionPkgUrl) throws IOException { + FunctionConfig functionConfig = new FunctionConfig(); + if (tenant != null) { + functionConfig.setTenant(tenant); + } + if (namespace != null) { + functionConfig.setNamespace(namespace); + } + if (function != null) { + functionConfig.setName(function); + } + if (topicsToSerDeClassName != null) { + functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); + } + if (outputTopic != null) { + functionConfig.setOutput(outputTopic); + } + if (outputSerdeClassName != null) { + functionConfig.setOutputSerdeClassName(outputSerdeClassName); + } + if (className != null) { + functionConfig.setClassName(className); + } + if (parallelism != null) { + functionConfig.setParallelism(parallelism); + } + functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); + + registerFunction(tenant, namespace, function, inputStream, details, functionPkgUrl, functionConfig); + + } + + @Test(expectedExceptions = Exception.class, expectedExceptionsMessageRegExp = "Function config is not provided") + public void testUpdateMissingFunctionConfig() throws IOException { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + updateFunction( + tenant, + namespace, + function, + mockedInputStream, + mockedFormData, + null, + null, + null, null); + } + + + private void registerDefaultFunction() throws IOException { + registerDefaultFunctionWithPackageUrl(null); + } + + private void registerDefaultFunctionWithPackageUrl(String packageUrl) throws IOException { + FunctionConfig functionConfig = createDefaultFunctionConfig(); + registerFunction(tenant, namespace, function, mockedInputStream, mockedFormData, packageUrl, functionConfig); + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function already" + + " exists") + public void testRegisterExistedFunction() throws IOException { + try { + Configurator.setRootLevel(Level.DEBUG); + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + registerDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure") + public void testRegisterFunctionUploadFailure() throws IOException { + try { + mockWorkerUtils(ctx -> { + ctx.when(() -> { + WorkerUtils.uploadFileToBookkeeper( + anyString(), + any(File.class), + any(Namespace.class)); + } + ).thenThrow(new IOException("upload failure")); + }); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); + + registerDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); + throw re; + } + } + + @Test + public void testRegisterFunctionSuccess() throws IOException { + try { + mockWorkerUtils(); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); + + registerDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(timeOut = 20000) + public void testRegisterFunctionSuccessWithPackageName() throws IOException { + registerDefaultFunctionWithPackageUrl("function://public/default/test@v1"); + } + + @Test(timeOut = 20000) + public void testRegisterFunctionFailedWithWrongPackageName() throws PulsarAdminException, IOException { + try { + doThrow(new PulsarAdminException("package name is invalid")) + .when(mockedPackages).download(anyString(), anyString()); + registerDefaultFunctionWithPackageUrl("function://"); + } catch (RestException e) { + // expected exception + assertEquals(e.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace does not exist") + public void testRegisterFunctionNonExistingNamespace() throws IOException { + try { + this.namespaceList.clear(); + registerDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant does not exist") + public void testRegisterFunctionNonexistantTenant() throws Exception { + try { + when(mockedTenants.getTenantInfo(any())).thenThrow(PulsarAdminException.NotFoundException.class); + registerDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to register") + public void testRegisterFunctionFailure() throws Exception { + try { + mockWorkerUtils(); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); + + doThrow(new IllegalArgumentException("function failed to register")) + .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); + + registerDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registration " + + "interrupted") + public void testRegisterFunctionInterrupted() throws Exception { + try { + mockWorkerUtils(); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); + + doThrow(new IllegalStateException("Function registration interrupted")) + .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); + + registerDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); + throw re; + } + } + + // + // Update Functions + // + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") + public void testUpdateFunctionMissingTenant() throws Exception { + try { + testUpdateFunctionMissingArguments( + null, + namespace, + function, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + className, + parallelism, + "Tenant is not provided"); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") + public void testUpdateFunctionMissingNamespace() throws Exception { + try { + testUpdateFunctionMissingArguments( + tenant, + null, + function, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + className, + parallelism, + "Namespace is not provided"); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") + public void testUpdateFunctionMissingFunctionName() throws Exception { + try { + testUpdateFunctionMissingArguments( + tenant, + namespace, + null, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + className, + parallelism, + "Function name is not provided"); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") + public void testUpdateFunctionMissingPackage() throws Exception { + try { + mockWorkerUtils(); + testUpdateFunctionMissingArguments( + tenant, + namespace, + function, + null, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + className, + parallelism, + "Update contains no change"); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") + public void testUpdateFunctionMissingInputTopic() throws Exception { + try { + mockWorkerUtils(); + + testUpdateFunctionMissingArguments( + tenant, + namespace, + function, + null, + null, + mockedFormData, + outputTopic, + outputSerdeClassName, + className, + parallelism, + "Update contains no change"); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") + public void testUpdateFunctionMissingClassName() throws Exception { + try { + mockWorkerUtils(); + + testUpdateFunctionMissingArguments( + tenant, + namespace, + function, + null, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + null, + parallelism, + "Update contains no change"); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test + public void testUpdateFunctionChangedParallelism() throws Exception { + try { + mockWorkerUtils(); + + testUpdateFunctionMissingArguments( + tenant, + namespace, + function, + null, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + null, + parallelism + 1, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test + public void testUpdateFunctionChangedInputs() throws Exception { + mockWorkerUtils(); + + testUpdateFunctionMissingArguments( + tenant, + namespace, + function, + null, + topicsToSerDeClassName, + mockedFormData, + "DifferentOutput", + outputSerdeClassName, + null, + parallelism, + null); + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Input Topics cannot be altered") + public void testUpdateFunctionChangedOutput() throws Exception { + try { + mockWorkerUtils(); + + Map someOtherInput = new HashMap<>(); + someOtherInput.put("DifferentTopic", TopicSchema.DEFAULT_SERDE); + testUpdateFunctionMissingArguments( + tenant, + namespace, + function, + null, + someOtherInput, + mockedFormData, + outputTopic, + outputSerdeClassName, + null, + parallelism, + "Input Topics cannot be altered"); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + private void testUpdateFunctionMissingArguments( + String tenant, + String namespace, + String function, + InputStream inputStream, + Map topicsToSerDeClassName, + FormDataContentDisposition details, + String outputTopic, + String outputSerdeClassName, + String className, + Integer parallelism, + String expectedError) throws Exception { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + FunctionConfig functionConfig = new FunctionConfig(); + if (tenant != null) { + functionConfig.setTenant(tenant); + } + if (namespace != null) { + functionConfig.setNamespace(namespace); + } + if (function != null) { + functionConfig.setName(function); + } + if (topicsToSerDeClassName != null) { + functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); + } + if (outputTopic != null) { + functionConfig.setOutput(outputTopic); + } + if (outputSerdeClassName != null) { + functionConfig.setOutputSerdeClassName(outputSerdeClassName); + } + if (className != null) { + functionConfig.setClassName(className); + } + if (parallelism != null) { + functionConfig.setParallelism(parallelism); + } + functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); + + if (expectedError != null) { + doThrow(new IllegalArgumentException(expectedError)) + .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); + } + + updateFunction( + tenant, + namespace, + function, + inputStream, + details, + null, + functionConfig, + null, null); + + } + + private void updateDefaultFunction() throws IOException { + updateDefaultFunctionWithPackageUrl(null); + } + + private void updateDefaultFunctionWithPackageUrl(String packageUrl) throws IOException { + FunctionConfig functionConfig = new FunctionConfig(); + functionConfig.setTenant(tenant); + functionConfig.setNamespace(namespace); + functionConfig.setName(function); + functionConfig.setClassName(className); + functionConfig.setParallelism(parallelism); + functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); + functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); + functionConfig.setOutput(outputTopic); + functionConfig.setOutputSerdeClassName(outputSerdeClassName); + + updateFunction( + tenant, + namespace, + function, + mockedInputStream, + mockedFormData, + packageUrl, + functionConfig, + null, null); + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't" + + " exist") + public void testUpdateNotExistedFunction() throws IOException { + try { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); + updateDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure") + public void testUpdateFunctionUploadFailure() throws Exception { + try { + mockWorkerUtils(ctx -> { + ctx.when(() -> { + WorkerUtils.uploadFileToBookkeeper( + anyString(), + any(File.class), + any(Namespace.class)); + + }).thenThrow(new IOException("upload failure")); + }); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + updateDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); + throw re; + } + } + + @Test + public void testUpdateFunctionSuccess() throws Exception { + mockWorkerUtils(); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + updateDefaultFunction(); + } + + @Test + public void testUpdateFunctionWithUrl() throws IOException { + Configurator.setRootLevel(Level.DEBUG); + + String fileLocation = FutureUtil.class.getProtectionDomain().getCodeSource().getLocation().getPath(); + String filePackageUrl = "file://" + fileLocation; + + FunctionConfig functionConfig = new FunctionConfig(); + functionConfig.setOutput(outputTopic); + functionConfig.setOutputSerdeClassName(outputSerdeClassName); + functionConfig.setTenant(tenant); + functionConfig.setNamespace(namespace); + functionConfig.setName(function); + functionConfig.setClassName(className); + functionConfig.setParallelism(parallelism); + functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); + functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + updateFunction( + tenant, + namespace, + function, + null, + null, + filePackageUrl, + functionConfig, + null, null); + + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to register") + public void testUpdateFunctionFailure() throws Exception { + try { + mockWorkerUtils(); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + doThrow(new IllegalArgumentException("function failed to register")) + .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); + + updateDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registeration " + + "interrupted") + public void testUpdateFunctionInterrupted() throws Exception { + try { + mockWorkerUtils(); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + doThrow(new IllegalStateException("Function registeration interrupted")) + .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); + + updateDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); + throw re; + } + } + + + @Test(timeOut = 20000) + public void testUpdateFunctionSuccessWithPackageName() throws IOException { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + updateDefaultFunctionWithPackageUrl("function://public/default/test@v1"); + } + + @Test(timeOut = 20000) + public void testUpdateFunctionFailedWithWrongPackageName() throws PulsarAdminException, IOException { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + try { + doThrow(new PulsarAdminException("package name is invalid")) + .when(mockedPackages).download(anyString(), anyString()); + registerDefaultFunctionWithPackageUrl("function://"); + } catch (RestException e) { + // expected exception + assertEquals(e.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + } + } + + // + // deregister function + // + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") + public void testDeregisterFunctionMissingTenant() { + try { + + testDeregisterFunctionMissingArguments( + null, + namespace, + function + ); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") + public void testDeregisterFunctionMissingNamespace() { + try { + testDeregisterFunctionMissingArguments( + tenant, + null, + function + ); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") + public void testDeregisterFunctionMissingFunctionName() { + try { + testDeregisterFunctionMissingArguments( + tenant, + namespace, + null + ); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't" + + " exist") + public void testDeregisterNotExistedFunction() { + try { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); + deregisterDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.NOT_FOUND); + throw re; + } + } + + @Test + public void testDeregisterFunctionSuccess() { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + deregisterDefaultFunction(); + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to deregister") + public void testDeregisterFunctionFailure() throws Exception { + try { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + doThrow(new IllegalArgumentException("function failed to deregister")) + .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); + + deregisterDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function deregisteration " + + "interrupted") + public void testDeregisterFunctionInterrupted() throws Exception { + try { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + doThrow(new IllegalStateException("Function deregisteration interrupted")) + .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); + + deregisterDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); + throw re; + } + } + + // + // Get Function Info + // + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") + public void testGetFunctionMissingTenant() throws IOException { + try { + testGetFunctionMissingArguments( + null, + namespace, + function + ); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") + public void testGetFunctionMissingNamespace() throws IOException { + try { + testGetFunctionMissingArguments( + tenant, + null, + function + ); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") + public void testGetFunctionMissingFunctionName() throws IOException { + try { + testGetFunctionMissingArguments( + tenant, + namespace, + null + ); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + // + // List Functions + // + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") + public void testListFunctionsMissingTenant() { + try { + testListFunctionsMissingArguments( + null, + namespace + ); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") + public void testListFunctionsMissingNamespace() { + try { + testListFunctionsMissingArguments( + tenant, + null + ); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test + public void testDownloadFunctionHttpUrl() throws Exception { + String jarHttpUrl = + "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar"; + File pkgFile = downloadFunction(jarHttpUrl, null); + pkgFile.delete(); + } + + @Test + public void testDownloadFunctionFile() throws Exception { + File file = getPulsarApiExamplesNar(); + File pkgFile = downloadFunction(file.toURI().toString(), null); + Assert.assertTrue(pkgFile.exists()); + Assert.assertEquals(file.length(), pkgFile.length()); + pkgFile.delete(); + } + + @Test + public void testDownloadFunctionBuiltinConnector() throws Exception { + File file = getPulsarApiExamplesNar(); + + WorkerConfig config = new WorkerConfig() + .setUploadBuiltinSinksSources(false); + when(mockedWorkerService.getWorkerConfig()).thenReturn(config); + + registerBuiltinConnector("cassandra", file); + + File pkgFile = downloadFunction("builtin://cassandra", null); + Assert.assertTrue(pkgFile.exists()); + Assert.assertEquals(file.length(), pkgFile.length()); + pkgFile.delete(); + } + + @Test + public void testDownloadFunctionBuiltinFunction() throws Exception { + File file = getPulsarApiExamplesNar(); + + WorkerConfig config = new WorkerConfig() + .setUploadBuiltinSinksSources(false); + when(mockedWorkerService.getWorkerConfig()).thenReturn(config); + + registerBuiltinFunction("exclamation", file); + + File pkgFile = downloadFunction("builtin://exclamation", null); + Assert.assertTrue(pkgFile.exists()); + Assert.assertEquals(file.length(), pkgFile.length()); + pkgFile.delete(); + } + + @Test + public void testRegisterFunctionFileUrlWithValidSinkClass() throws Exception { + Configurator.setRootLevel(Level.DEBUG); + + File file = getPulsarApiExamplesNar(); + String filePackageUrl = file.toURI().toString(); + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); + + FunctionConfig functionConfig = new FunctionConfig(); + functionConfig.setTenant(tenant); + functionConfig.setNamespace(namespace); + functionConfig.setName(function); + functionConfig.setClassName(className); + functionConfig.setParallelism(parallelism); + functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); + functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); + functionConfig.setOutput(outputTopic); + functionConfig.setOutputSerdeClassName(outputSerdeClassName); + registerFunction(tenant, namespace, function, null, null, filePackageUrl, functionConfig); + + } + + @Test + public void testRegisterFunctionWithConflictingFields() throws Exception { + Configurator.setRootLevel(Level.DEBUG); + String actualTenant = "DIFFERENT_TENANT"; + String actualNamespace = "DIFFERENT_NAMESPACE"; + String actualName = "DIFFERENT_NAME"; + this.namespaceList.add(actualTenant + "/" + actualNamespace); + + File file = getPulsarApiExamplesNar(); + String filePackageUrl = file.toURI().toString(); + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + when(mockedManager.containsFunction(eq(actualTenant), eq(actualNamespace), eq(actualName))).thenReturn(false); + + FunctionConfig functionConfig = new FunctionConfig(); + functionConfig.setTenant(tenant); + functionConfig.setNamespace(namespace); + functionConfig.setName(function); + functionConfig.setClassName(className); + functionConfig.setParallelism(parallelism); + functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); + functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); + functionConfig.setOutput(outputTopic); + functionConfig.setOutputSerdeClassName(outputSerdeClassName); + registerFunction(actualTenant, actualNamespace, actualName, null, null, filePackageUrl, functionConfig); + } + + public static FunctionConfig createDefaultFunctionConfig() { + FunctionConfig functionConfig = new FunctionConfig(); + functionConfig.setTenant(tenant); + functionConfig.setNamespace(namespace); + functionConfig.setName(function); + functionConfig.setClassName(className); + functionConfig.setParallelism(parallelism); + functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); + functionConfig.setOutput(outputTopic); + functionConfig.setOutputSerdeClassName(outputSerdeClassName); + functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); + return functionConfig; + } + + public static FunctionDetails createDefaultFunctionDetails() { + FunctionConfig functionConfig = createDefaultFunctionConfig(); + return FunctionConfigUtils.convert(functionConfig); + } +} diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionsResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionsResourceTest.java new file mode 100644 index 0000000000000..51ca4c83f9e02 --- /dev/null +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionsResourceTest.java @@ -0,0 +1,337 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.worker.rest.api.v3; + +import static org.apache.pulsar.functions.source.TopicSchema.DEFAULT_SERDE; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.lang.reflect.Method; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Consumer; +import org.apache.distributedlog.api.namespace.Namespace; +import org.apache.pulsar.client.admin.Functions; +import org.apache.pulsar.client.admin.Namespaces; +import org.apache.pulsar.client.admin.Packages; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.Tenants; +import org.apache.pulsar.common.functions.FunctionDefinition; +import org.apache.pulsar.common.io.ConnectorDefinition; +import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.functions.instance.InstanceUtils; +import org.apache.pulsar.functions.proto.Function; +import org.apache.pulsar.functions.runtime.RuntimeFactory; +import org.apache.pulsar.functions.source.TopicSchema; +import org.apache.pulsar.functions.utils.LoadedFunctionPackage; +import org.apache.pulsar.functions.utils.ValidatableFunctionPackage; +import org.apache.pulsar.functions.utils.functions.FunctionArchive; +import org.apache.pulsar.functions.utils.functions.FunctionUtils; +import org.apache.pulsar.functions.utils.io.Connector; +import org.apache.pulsar.functions.utils.io.ConnectorUtils; +import org.apache.pulsar.functions.worker.ConnectorsManager; +import org.apache.pulsar.functions.worker.FunctionMetaDataManager; +import org.apache.pulsar.functions.worker.FunctionRuntimeManager; +import org.apache.pulsar.functions.worker.FunctionsManager; +import org.apache.pulsar.functions.worker.LeaderService; +import org.apache.pulsar.functions.worker.PackageUrlValidator; +import org.apache.pulsar.functions.worker.PulsarWorkerService; +import org.apache.pulsar.functions.worker.WorkerConfig; +import org.apache.pulsar.functions.worker.WorkerUtils; +import org.apache.pulsar.functions.worker.rest.api.PulsarFunctionTestTemporaryDirectory; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.mockito.Answers; +import org.mockito.MockSettings; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; + +public abstract class AbstractFunctionsResourceTest { + + protected static final String tenant = "test-tenant"; + protected static final String namespace = "test-namespace"; + protected static final Map topicsToSerDeClassName = new HashMap<>(); + protected static final String subscriptionName = "test-subscription"; + protected static final String CASSANDRA_STRING_SINK = "org.apache.pulsar.io.cassandra.CassandraStringSink"; + protected static final int parallelism = 1; + private static final String SYSTEM_PROPERTY_NAME_CASSANDRA_NAR_FILE_PATH = "pulsar-io-cassandra.nar.path"; + private static final String SYSTEM_PROPERTY_NAME_TWITTER_NAR_FILE_PATH = "pulsar-io-twitter.nar.path"; + private static final String SYSTEM_PROPERTY_NAME_INVALID_NAR_FILE_PATH = "pulsar-io-invalid.nar.path"; + private static final String SYSTEM_PROPERTY_NAME_FUNCTIONS_API_EXAMPLES_NAR_FILE_PATH = + "pulsar-functions-api-examples.nar.path"; + protected static Map mockStaticContexts = new HashMap<>(); + + static { + topicsToSerDeClassName.put("test_src", DEFAULT_SERDE); + topicsToSerDeClassName.put("persistent://public/default/test_src", TopicSchema.DEFAULT_SERDE); + } + + protected PulsarWorkerService mockedWorkerService; + protected PulsarAdmin mockedPulsarAdmin; + protected Tenants mockedTenants; + protected Namespaces mockedNamespaces; + protected Functions mockedFunctions; + protected TenantInfoImpl mockedTenantInfo; + protected List namespaceList = new LinkedList<>(); + protected FunctionMetaDataManager mockedManager; + protected FunctionRuntimeManager mockedFunctionRunTimeManager; + protected RuntimeFactory mockedRuntimeFactory; + protected Namespace mockedNamespace; + protected InputStream mockedInputStream; + protected FormDataContentDisposition mockedFormData; + protected Function.FunctionMetaData mockedFunctionMetaData; + protected LeaderService mockedLeaderService; + protected Packages mockedPackages; + protected PulsarFunctionTestTemporaryDirectory tempDirectory; + protected ConnectorsManager connectorsManager = new ConnectorsManager(); + protected FunctionsManager functionsManager = new FunctionsManager(); + + public static File getPulsarIOCassandraNar() { + return new File(Objects.requireNonNull(System.getProperty(SYSTEM_PROPERTY_NAME_CASSANDRA_NAR_FILE_PATH) + , "pulsar-io-cassandra.nar file location must be specified with " + + SYSTEM_PROPERTY_NAME_CASSANDRA_NAR_FILE_PATH + " system property")); + } + + public static File getPulsarIOTwitterNar() { + return new File(Objects.requireNonNull(System.getProperty(SYSTEM_PROPERTY_NAME_TWITTER_NAR_FILE_PATH) + , "pulsar-io-twitter.nar file location must be specified with " + + SYSTEM_PROPERTY_NAME_TWITTER_NAR_FILE_PATH + " system property")); + } + + public static File getPulsarIOInvalidNar() { + return new File(Objects.requireNonNull(System.getProperty(SYSTEM_PROPERTY_NAME_INVALID_NAR_FILE_PATH) + , "invalid nar file location must be specified with " + + SYSTEM_PROPERTY_NAME_INVALID_NAR_FILE_PATH + " system property")); + } + + public static File getPulsarApiExamplesNar() { + return new File(Objects.requireNonNull( + System.getProperty(SYSTEM_PROPERTY_NAME_FUNCTIONS_API_EXAMPLES_NAR_FILE_PATH) + , "pulsar-functions-api-examples.nar file location must be specified with " + + SYSTEM_PROPERTY_NAME_FUNCTIONS_API_EXAMPLES_NAR_FILE_PATH + " system property")); + } + + @BeforeMethod + public final void setup(Method method) throws Exception { + this.mockedManager = mock(FunctionMetaDataManager.class); + this.mockedFunctionRunTimeManager = mock(FunctionRuntimeManager.class); + this.mockedRuntimeFactory = mock(RuntimeFactory.class); + this.mockedInputStream = mock(InputStream.class); + this.mockedNamespace = mock(Namespace.class); + this.mockedFormData = mock(FormDataContentDisposition.class); + when(mockedFormData.getFileName()).thenReturn("test"); + this.mockedTenantInfo = mock(TenantInfoImpl.class); + this.mockedPulsarAdmin = mock(PulsarAdmin.class); + this.mockedTenants = mock(Tenants.class); + this.mockedNamespaces = mock(Namespaces.class); + this.mockedFunctions = mock(Functions.class); + this.mockedLeaderService = mock(LeaderService.class); + this.mockedPackages = mock(Packages.class); + namespaceList.add(tenant + "/" + namespace); + + this.mockedWorkerService = mock(PulsarWorkerService.class); + when(mockedWorkerService.getFunctionMetaDataManager()).thenReturn(mockedManager); + when(mockedWorkerService.getLeaderService()).thenReturn(mockedLeaderService); + when(mockedWorkerService.getFunctionRuntimeManager()).thenReturn(mockedFunctionRunTimeManager); + when(mockedFunctionRunTimeManager.getRuntimeFactory()).thenReturn(mockedRuntimeFactory); + when(mockedWorkerService.getDlogNamespace()).thenReturn(mockedNamespace); + when(mockedWorkerService.isInitialized()).thenReturn(true); + when(mockedWorkerService.getBrokerAdmin()).thenReturn(mockedPulsarAdmin); + when(mockedWorkerService.getFunctionAdmin()).thenReturn(mockedPulsarAdmin); + when(mockedPulsarAdmin.tenants()).thenReturn(mockedTenants); + when(mockedPulsarAdmin.namespaces()).thenReturn(mockedNamespaces); + when(mockedPulsarAdmin.functions()).thenReturn(mockedFunctions); + when(mockedPulsarAdmin.packages()).thenReturn(mockedPackages); + when(mockedTenants.getTenantInfo(any())).thenReturn(mockedTenantInfo); + when(mockedNamespaces.getNamespaces(any())).thenReturn(namespaceList); + when(mockedLeaderService.isLeader()).thenReturn(true); + doAnswer(invocationOnMock -> { + Files.copy(getDefaultNarFile().toPath(), Paths.get(invocationOnMock.getArgument(1, String.class)), + StandardCopyOption.REPLACE_EXISTING); + return null; + }).when(mockedPackages).download(any(), any()); + + // worker config + List urlPatterns = + List.of("http://localhost.*", "file:.*", "https://repo1.maven.org/maven2/org/apache/pulsar/.*"); + WorkerConfig workerConfig = new WorkerConfig() + .setWorkerId("test") + .setWorkerPort(8080) + .setFunctionMetadataTopicName("pulsar/functions") + .setNumFunctionPackageReplicas(3) + .setPulsarServiceUrl("pulsar://localhost:6650/") + .setAdditionalEnabledFunctionsUrlPatterns(urlPatterns) + .setAdditionalEnabledConnectorUrlPatterns(urlPatterns) + .setFunctionsWorkerEnablePackageManagement(true); + customizeWorkerConfig(workerConfig, method); + tempDirectory = PulsarFunctionTestTemporaryDirectory.create(getClass().getSimpleName()); + tempDirectory.useTemporaryDirectoriesForWorkerConfig(workerConfig); + when(mockedWorkerService.getWorkerConfig()).thenReturn(workerConfig); + when(mockedWorkerService.getFunctionsManager()).thenReturn(functionsManager); + when(mockedWorkerService.getConnectorsManager()).thenReturn(connectorsManager); + PackageUrlValidator packageUrlValidator = new PackageUrlValidator(workerConfig); + when(mockedWorkerService.getPackageUrlValidator()).thenReturn(packageUrlValidator); + + doSetup(); + } + + protected void customizeWorkerConfig(WorkerConfig workerConfig, Method method) { + + } + + protected File getDefaultNarFile() { + return getPulsarIOTwitterNar(); + } + + protected void doSetup() throws Exception { + + } + + @AfterMethod(alwaysRun = true) + public void cleanup() { + if (tempDirectory != null) { + tempDirectory.delete(); + } + mockStaticContexts.values().forEach(MockedStatic::close); + mockStaticContexts.clear(); + } + + protected void mockStatic(Class classStatic, Consumer> consumer) { + mockStatic(classStatic, withSettings().defaultAnswer(Mockito.CALLS_REAL_METHODS), consumer); + } + + private void mockStatic(Class classStatic, MockSettings mockSettings, Consumer> consumer) { + final MockedStatic mockedStatic = mockStaticContexts.computeIfAbsent(classStatic.getName(), + name -> Mockito.mockStatic(classStatic, mockSettings)); + consumer.accept(mockedStatic); + } + + protected void mockWorkerUtils() { + mockWorkerUtils(null); + } + + protected void mockWorkerUtils(Consumer> consumer) { + mockStatic(WorkerUtils.class, withSettings(), ctx -> { + // make dumpToTmpFile return the nar file copy + ctx.when(() -> WorkerUtils.dumpToTmpFile(mockedInputStream)) + .thenAnswer(invocation -> { + Path tempFile = Files.createTempFile("test", ".nar"); + Files.copy(getPulsarApiExamplesNar().toPath(), tempFile, + StandardCopyOption.REPLACE_EXISTING); + return tempFile.toFile(); + }); + ctx.when(() -> WorkerUtils.dumpToTmpFile(any())) + .thenAnswer(Answers.CALLS_REAL_METHODS); + if (consumer != null) { + consumer.accept(ctx); + } + }); + } + + protected void mockInstanceUtils() { + mockStatic(InstanceUtils.class, ctx -> { + ctx.when(() -> InstanceUtils.calculateSubjectType(any())) + .thenReturn(getComponentType()); + }); + } + + protected abstract Function.FunctionDetails.ComponentType getComponentType(); + + public static class LoadedConnector extends Connector { + public LoadedConnector(ConnectorDefinition connectorDefinition) { + super(null, connectorDefinition, null, true); + } + + @Override + public ValidatableFunctionPackage getConnectorFunctionPackage() { + return new LoadedFunctionPackage(getClass().getClassLoader(), ConnectorDefinition.class, + getConnectorDefinition()); + } + } + + + protected void registerBuiltinConnector(String connectorType, String className) { + ConnectorDefinition connectorDefinition = null; + if (className != null) { + connectorDefinition = new ConnectorDefinition(); + // set source and sink class to the same to simplify the test + connectorDefinition.setSinkClass(className); + connectorDefinition.setSourceClass(className); + } + connectorsManager.addConnector(connectorType, new LoadedConnector(connectorDefinition)); + } + + protected void registerBuiltinConnector(String connectorType, File packageFile) { + ConnectorDefinition cntDef; + try { + cntDef = ConnectorUtils.getConnectorDefinition(packageFile); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + connectorsManager.addConnector(connectorType, + new Connector(packageFile.toPath(), cntDef, NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, true)); + } + + public static class LoadedFunctionArchive extends FunctionArchive { + public LoadedFunctionArchive(FunctionDefinition functionDefinition) { + super(null, functionDefinition, null, true); + } + + @Override + public ValidatableFunctionPackage getFunctionPackage() { + return new LoadedFunctionPackage(getClass().getClassLoader(), FunctionDefinition.class, + getFunctionDefinition()); + } + } + + protected void registerBuiltinFunction(String functionType, String className) { + FunctionDefinition functionDefinition = null; + if (className != null) { + functionDefinition = new FunctionDefinition(); + functionDefinition.setFunctionClass(className); + } + functionsManager.addFunction(functionType, new LoadedFunctionArchive(functionDefinition)); + } + + protected void registerBuiltinFunction(String functionType, File packageFile) { + FunctionDefinition cntDef; + try { + cntDef = FunctionUtils.getFunctionDefinition(packageFile); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + functionsManager.addFunction(functionType, + new FunctionArchive(packageFile.toPath(), cntDef, NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, true)); + } +} diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java index 0c20083bb89ca..a1a418460be45 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java @@ -18,1791 +18,168 @@ */ package org.apache.pulsar.functions.worker.rest.api.v3; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -import com.google.common.collect.Lists; import java.io.File; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.net.URL; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.LinkedList; import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.UUID; -import java.util.function.Consumer; import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; import org.apache.distributedlog.api.namespace.Namespace; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; import org.apache.pulsar.broker.authentication.AuthenticationParameters; -import org.apache.pulsar.client.admin.Functions; -import org.apache.pulsar.client.admin.Namespaces; -import org.apache.pulsar.client.admin.Packages; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.admin.Tenants; import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.functions.UpdateOptionsImpl; -import org.apache.pulsar.common.nar.NarClassLoader; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; -import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.RestException; -import org.apache.pulsar.functions.api.Context; -import org.apache.pulsar.functions.api.Function; -import org.apache.pulsar.functions.instance.InstanceUtils; -import org.apache.pulsar.functions.proto.Function.FunctionDetails; -import org.apache.pulsar.functions.proto.Function.FunctionMetaData; -import org.apache.pulsar.functions.proto.Function.PackageLocationMetaData; -import org.apache.pulsar.functions.proto.Function.ProcessingGuarantees; -import org.apache.pulsar.functions.proto.Function.SinkSpec; -import org.apache.pulsar.functions.proto.Function.SourceSpec; -import org.apache.pulsar.functions.proto.Function.SubscriptionType; -import org.apache.pulsar.functions.runtime.RuntimeFactory; -import org.apache.pulsar.functions.source.TopicSchema; -import org.apache.pulsar.functions.utils.FunctionCommon; +import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.utils.FunctionConfigUtils; -import org.apache.pulsar.functions.utils.functions.FunctionArchive; -import org.apache.pulsar.functions.utils.io.Connector; -import org.apache.pulsar.functions.worker.ConnectorsManager; -import org.apache.pulsar.functions.worker.FunctionMetaDataManager; -import org.apache.pulsar.functions.worker.FunctionRuntimeManager; -import org.apache.pulsar.functions.worker.FunctionsManager; -import org.apache.pulsar.functions.worker.LeaderService; -import org.apache.pulsar.functions.worker.PulsarWorkerService; import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerUtils; import org.apache.pulsar.functions.worker.rest.api.FunctionsImpl; -import org.apache.pulsar.functions.worker.rest.api.PulsarFunctionTestTemporaryDirectory; -import org.apache.pulsar.functions.worker.rest.api.v2.FunctionsApiV2Resource; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; -import org.mockito.MockedStatic; -import org.mockito.Mockito; import org.testng.Assert; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -/** - * Unit test of {@link FunctionsApiV2Resource}. - */ -public class FunctionApiV3ResourceTest { - - private static final class TestFunction implements Function { - - @Override - public String process(String input, Context context) { - return input; - } - } - - private static final class WrongFunction implements Consumer { - @Override - public void accept(String s) { - - } - } - - private static final String tenant = "test-tenant"; - private static final String namespace = "test-namespace"; - private static final String function = "test-function"; - private static final String outputTopic = "test-output-topic"; - private static final String outputSerdeClassName = TopicSchema.DEFAULT_SERDE; - private static final String className = TestFunction.class.getName(); - private SubscriptionType subscriptionType = SubscriptionType.FAILOVER; - private static final Map topicsToSerDeClassName = new HashMap<>(); - static { - topicsToSerDeClassName.put("persistent://public/default/test_src", TopicSchema.DEFAULT_SERDE); - } - private static final int parallelism = 1; - - private PulsarWorkerService mockedWorkerService; - private PulsarAdmin mockedPulsarAdmin; - private Tenants mockedTenants; - private Namespaces mockedNamespaces; - private Functions mockedFunctions; - private TenantInfoImpl mockedTenantInfo; - private List namespaceList = new LinkedList<>(); - private FunctionMetaDataManager mockedManager; - private FunctionRuntimeManager mockedFunctionRunTimeManager; - private RuntimeFactory mockedRuntimeFactory; - private Namespace mockedNamespace; +public class FunctionApiV3ResourceTest extends AbstractFunctionApiResourceTest { private FunctionsImpl resource; - private InputStream mockedInputStream; - private FormDataContentDisposition mockedFormData; - private FunctionMetaData mockedFunctionMetadata; - private LeaderService mockedLeaderService; - private Packages mockedPackages; - private PulsarFunctionTestTemporaryDirectory tempDirectory; - private static Map mockStaticContexts = new HashMap<>(); - - private static final String SYSTEM_PROPERTY_NAME_FUNCTIONS_API_EXAMPLES_NAR_FILE_PATH = - "pulsar-functions-api-examples.nar.path"; - - public static File getPulsarApiExamplesNar() { - return new File(Objects.requireNonNull( - System.getProperty(SYSTEM_PROPERTY_NAME_FUNCTIONS_API_EXAMPLES_NAR_FILE_PATH) - , "pulsar-functions-api-examples.nar file location must be specified with " - + SYSTEM_PROPERTY_NAME_FUNCTIONS_API_EXAMPLES_NAR_FILE_PATH + " system property")); - } - - @BeforeMethod - public void setup() throws Exception { - this.mockedManager = mock(FunctionMetaDataManager.class); - this.mockedFunctionRunTimeManager = mock(FunctionRuntimeManager.class); - this.mockedTenantInfo = mock(TenantInfoImpl.class); - this.mockedRuntimeFactory = mock(RuntimeFactory.class); - this.mockedInputStream = mock(InputStream.class); - this.mockedNamespace = mock(Namespace.class); - this.mockedFormData = mock(FormDataContentDisposition.class); - when(mockedFormData.getFileName()).thenReturn("test"); - this.mockedPulsarAdmin = mock(PulsarAdmin.class); - this.mockedTenants = mock(Tenants.class); - this.mockedNamespaces = mock(Namespaces.class); - this.mockedFunctions = mock(Functions.class); - this.mockedPackages = mock(Packages.class); - this.mockedLeaderService = mock(LeaderService.class); - this.mockedFunctionMetadata = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); - namespaceList.add(tenant + "/" + namespace); - - this.mockedWorkerService = mock(PulsarWorkerService.class); - when(mockedWorkerService.getFunctionMetaDataManager()).thenReturn(mockedManager); - when(mockedWorkerService.getFunctionRuntimeManager()).thenReturn(mockedFunctionRunTimeManager); - when(mockedWorkerService.getLeaderService()).thenReturn(mockedLeaderService); - when(mockedFunctionRunTimeManager.getRuntimeFactory()).thenReturn(mockedRuntimeFactory); - when(mockedWorkerService.getDlogNamespace()).thenReturn(mockedNamespace); - when(mockedWorkerService.isInitialized()).thenReturn(true); - when(mockedWorkerService.getBrokerAdmin()).thenReturn(mockedPulsarAdmin); - when(mockedWorkerService.getFunctionAdmin()).thenReturn(mockedPulsarAdmin); - when(mockedPulsarAdmin.tenants()).thenReturn(mockedTenants); - when(mockedPulsarAdmin.namespaces()).thenReturn(mockedNamespaces); - when(mockedPulsarAdmin.functions()).thenReturn(mockedFunctions); - when(mockedPulsarAdmin.packages()).thenReturn(mockedPackages); - when(mockedTenants.getTenantInfo(any())).thenReturn(mockedTenantInfo); - when(mockedNamespaces.getNamespaces(any())).thenReturn(namespaceList); - when(mockedLeaderService.isLeader()).thenReturn(true); - when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetadata); - doNothing().when(mockedPackages).download(anyString(), anyString()); - - // worker config - WorkerConfig workerConfig = new WorkerConfig() - .setWorkerId("test") - .setWorkerPort(8080) - .setFunctionMetadataTopicName("pulsar/functions") - .setNumFunctionPackageReplicas(3) - .setPulsarServiceUrl("pulsar://localhost:6650/"); - tempDirectory = PulsarFunctionTestTemporaryDirectory.create(getClass().getSimpleName()); - tempDirectory.useTemporaryDirectoriesForWorkerConfig(workerConfig); - when(mockedWorkerService.getWorkerConfig()).thenReturn(workerConfig); - + @Override + protected void doSetup() { + super.doSetup(); this.resource = spy(new FunctionsImpl(() -> mockedWorkerService)); } - @AfterMethod(alwaysRun = true) - public void cleanup() { - if (tempDirectory != null) { - tempDirectory.delete(); - } - mockStaticContexts.values().forEach(MockedStatic::close); - mockStaticContexts.clear(); - } - - private void mockStatic(Class classStatic, Consumer> consumer) { - final MockedStatic mockedStatic = mockStaticContexts.computeIfAbsent(classStatic.getName(), name -> Mockito.mockStatic(classStatic)); - consumer.accept(mockedStatic); - } - - private void mockWorkerUtils() { - mockWorkerUtils(null); - } - - private void mockWorkerUtils(Consumer> consumer) { - mockStatic(WorkerUtils.class, ctx -> { - ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); - if (consumer != null) { - consumer.accept(ctx); - } - }); - } - - private void mockInstanceUtils() { - mockStatic(InstanceUtils.class, ctx -> { - ctx.when(() -> InstanceUtils.calculateSubjectType(any())) - .thenReturn(FunctionDetails.ComponentType.FUNCTION); - }); - } - - // - // Register Functions - // - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") - public void testRegisterFunctionMissingTenant() { - try { - testRegisterFunctionMissingArguments( - null, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") - public void testRegisterFunctionMissingNamespace() { - try { - testRegisterFunctionMissingArguments( - tenant, - null, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re){ - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") - public void testRegisterFunctionMissingFunctionName() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - null, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re){ - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function package is not provided") - public void testRegisterFunctionMissingPackage() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - null, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "No input topic\\(s\\) specified for the function") - public void testRegisterFunctionMissingInputTopics() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - null, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function Package is not provided") - public void testRegisterFunctionMissingPackageDetails() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - null, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re){ - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function package does not have" - + " the correct format. Pulsar cannot determine if the package is a NAR package or JAR package. Function " - + "classname is not provided and attempts to load it as a NAR package produced the following error.*") - public void testRegisterFunctionMissingClassName() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - null, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function class UnknownClass must be in class path") - public void testRegisterFunctionWrongClassName() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - "UnknownClass", - parallelism, - null); - } catch (RestException re){ - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function parallelism must be a positive number") - public void testRegisterFunctionWrongParallelism() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - -2, - null); - } catch (RestException re){ - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, - expectedExceptionsMessageRegExp = "Output topic persistent://public/default/test_src is also being used as an input topic \\(topics must be one or the other\\)") - public void testRegisterFunctionSameInputOutput() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - topicsToSerDeClassName.keySet().iterator().next(), - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Output topic " + function + "-output-topic/test:" + " is invalid") - public void testRegisterFunctionWrongOutputTopic() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - function + "-output-topic/test:", - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re){ - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Encountered error .*. when getting Function package from .*") - public void testRegisterFunctionHttpUrl() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - null, - topicsToSerDeClassName, - null, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "http://localhost:1234/test"); - } catch (RestException re){ - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function class .*. does not implement the correct interface") - public void testRegisterFunctionImplementWrongInterface() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - WrongFunction.class.getName(), - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - private void testRegisterFunctionMissingArguments( - String tenant, - String namespace, - String function, - InputStream inputStream, - Map topicsToSerDeClassName, - FormDataContentDisposition details, - String outputTopic, - String outputSerdeClassName, - String className, - Integer parallelism, - String functionPkgUrl) { - FunctionConfig functionConfig = new FunctionConfig(); - if (tenant != null) { - functionConfig.setTenant(tenant); - } - if (namespace != null) { - functionConfig.setNamespace(namespace); - } - if (function != null) { - functionConfig.setName(function); - } - if (topicsToSerDeClassName != null) { - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - } - if (outputTopic != null) { - functionConfig.setOutput(outputTopic); - } - if (outputSerdeClassName != null) { - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - } - if (className != null) { - functionConfig.setClassName(className); - } - if (parallelism != null) { - functionConfig.setParallelism(parallelism); - } - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - - resource.registerFunction( - tenant, - namespace, - function, - inputStream, - details, - functionPkgUrl, - functionConfig, - null); - - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function config is not provided") - public void testMissingFunctionConfig() { - resource.registerFunction( - tenant, - namespace, - function, - mockedInputStream, - mockedFormData, - null, - null, - null); - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function config is not provided") - public void testUpdateMissingFunctionConfig() { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - resource.updateFunction( - tenant, - namespace, - function, - mockedInputStream, - mockedFormData, - null, - null, - null, null); - } - - @Test - public void testUpdateSourceWithNoChange() throws ClassNotFoundException { - mockWorkerUtils(); - - FunctionDetails functionDetails = createDefaultFunctionDetails(); - NarClassLoader mockedClassLoader = mock(NarClassLoader.class); - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.getFunctionTypes(any(FunctionConfig.class), any(Class.class))).thenReturn(new Class[]{String.class, String.class}); - ctx.when(() -> FunctionCommon.convertRuntime(any(FunctionConfig.Runtime.class))).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.isFunctionCodeBuiltin(any())).thenReturn(true); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(),any(),any(),any())).thenCallRealMethod(); - ctx.when(FunctionCommon::createPkgTempFile).thenCallRealMethod(); - }); - - doReturn(Function.class).when(mockedClassLoader).loadClass(anyString()); - - FunctionsManager mockedFunctionsManager = mock(FunctionsManager.class); - FunctionArchive functionArchive = FunctionArchive.builder() - .classLoader(mockedClassLoader) - .build(); - when(mockedFunctionsManager.getFunction("exclamation")).thenReturn(functionArchive); - when(mockedFunctionsManager.getFunctionArchive(any())).thenReturn(getPulsarApiExamplesNar().toPath()); - - when(mockedWorkerService.getFunctionsManager()).thenReturn(mockedFunctionsManager); - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - // No change on config, - FunctionConfig funcConfig = createDefaultFunctionConfig(); - mockStatic(FunctionConfigUtils.class, ctx -> { - ctx.when(() -> FunctionConfigUtils.convertFromDetails(any())).thenReturn(funcConfig); - ctx.when(() -> FunctionConfigUtils.validateUpdate(any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionConfigUtils.convert(any(FunctionConfig.class), any(ClassLoader.class))).thenReturn(functionDetails); - ctx.when(() -> FunctionConfigUtils.convert(any(FunctionConfig.class), any(FunctionConfigUtils.ExtractedFunctionDetails.class))).thenReturn(functionDetails); - ctx.when(() -> FunctionConfigUtils.validateJavaFunction(any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionConfigUtils.doCommonChecks(any())).thenCallRealMethod(); - ctx.when(() -> FunctionConfigUtils.collectAllInputTopics(any())).thenCallRealMethod(); - ctx.when(() -> FunctionConfigUtils.doJavaChecks(any(), any())).thenCallRealMethod(); - }); - - // config has not changes and don't update auth, should fail - try { - resource.updateFunction( - funcConfig.getTenant(), - funcConfig.getNamespace(), - funcConfig.getName(), - null, - mockedFormData, - null, - funcConfig, - null, - null); - fail("Update without changes should fail"); - } catch (RestException e) { - assertTrue(e.getMessage().contains("Update contains no change")); - } - - try { - UpdateOptionsImpl updateOptions = new UpdateOptionsImpl(); - updateOptions.setUpdateAuthData(false); - resource.updateFunction( - funcConfig.getTenant(), - funcConfig.getNamespace(), - funcConfig.getName(), - null, - mockedFormData, - null, - funcConfig, - null, - updateOptions); - fail("Update without changes should fail"); - } catch (RestException e) { - assertTrue(e.getMessage().contains("Update contains no change")); - } - - // no changes but set the auth-update flag to true, should not fail - UpdateOptionsImpl updateOptions = new UpdateOptionsImpl(); - updateOptions.setUpdateAuthData(true); - resource.updateFunction( - funcConfig.getTenant(), - funcConfig.getNamespace(), - funcConfig.getName(), - null, - mockedFormData, - null, - funcConfig, - null, - updateOptions); - } - - - private void registerDefaultFunction() { - registerDefaultFunctionWithPackageUrl(null); - } - - private void registerDefaultFunctionWithPackageUrl(String packageUrl) { - FunctionConfig functionConfig = createDefaultFunctionConfig(); - resource.registerFunction( - tenant, - namespace, - function, - mockedInputStream, - mockedFormData, - packageUrl, - functionConfig, - null); - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function already exists") - public void testRegisterExistedFunction() { - try { - Configurator.setRootLevel(Level.DEBUG); - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure") - public void testRegisterFunctionUploadFailure() throws Exception { - try { - mockWorkerUtils(ctx -> { - ctx.when(() -> { - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - } - ).thenThrow(new IOException("upload failure")); - ; - }); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); - throw re; - } - } - - @Test - public void testRegisterFunctionSuccess() throws Exception { - try { - mockWorkerUtils(); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(timeOut = 20000) - public void testRegisterFunctionSuccessWithPackageName() { - registerDefaultFunctionWithPackageUrl("function://public/default/test@v1"); - } - - @Test(timeOut = 20000) - public void testRegisterFunctionFailedWithWrongPackageName() throws PulsarAdminException { - try { - doThrow(new PulsarAdminException("package name is invalid")) - .when(mockedPackages).download(anyString(), anyString()); - registerDefaultFunctionWithPackageUrl("function://"); - } catch (RestException e) { - // expected exception - assertEquals(e.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace does not exist") - public void testRegisterFunctionNonExistingNamespace() { - try { - this.namespaceList.clear(); - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant does not exist") - public void testRegisterFunctionNonexistantTenant() throws Exception { - try { - when(mockedTenants.getTenantInfo(any())).thenThrow(PulsarAdminException.NotFoundException.class); - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to register") - public void testRegisterFunctionFailure() throws Exception { - try { - mockWorkerUtils(); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - - doThrow(new IllegalArgumentException("function failed to register")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); - - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registration interrupted") - public void testRegisterFunctionInterrupted() throws Exception { - try { - mockWorkerUtils(); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - - doThrow(new IllegalStateException("Function registration interrupted")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); - - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); - throw re; - } - } - - /* - Externally managed runtime, - uploadBuiltinSinksSources == false - Make sure uploadFileToBookkeeper is not called - */ - @Test - public void testRegisterFunctionSuccessK8sNoUpload() throws Exception { - mockedWorkerService.getWorkerConfig().setUploadBuiltinSinksSources(false); - - mockStatic(WorkerUtils.class, ctx -> { - ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class))) - .thenThrow(new RuntimeException("uploadFileToBookkeeper triggered")); - - }); - - NarClassLoader mockedClassLoader = mock(NarClassLoader.class); - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.getFunctionTypes(any(FunctionConfig.class), any(Class.class))).thenReturn(new Class[]{String.class, String.class}); - ctx.when(() -> FunctionCommon.convertRuntime(any(FunctionConfig.Runtime.class))).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.isFunctionCodeBuiltin(any())).thenReturn(true); - - }); - - doReturn(Function.class).when(mockedClassLoader).loadClass(anyString()); - - FunctionsManager mockedFunctionsManager = mock(FunctionsManager.class); - FunctionArchive functionArchive = FunctionArchive.builder() - .classLoader(mockedClassLoader) - .build(); - when(mockedFunctionsManager.getFunction("exclamation")).thenReturn(functionArchive); - when(mockedFunctionsManager.getFunctionArchive(any())).thenReturn(getPulsarApiExamplesNar().toPath()); - - when(mockedWorkerService.getFunctionsManager()).thenReturn(mockedFunctionsManager); - - when(mockedRuntimeFactory.externallyManaged()).thenReturn(true); - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - - FunctionConfig functionConfig = createDefaultFunctionConfig(); - functionConfig.setJar("builtin://exclamation"); - - try (FileInputStream inputStream = new FileInputStream(getPulsarApiExamplesNar())) { - resource.registerFunction( - tenant, - namespace, - function, - inputStream, - mockedFormData, - null, - functionConfig, - null); - } - } - - /* - Externally managed runtime, - uploadBuiltinSinksSources == true - Make sure uploadFileToBookkeeper is called - */ - @Test - public void testRegisterFunctionSuccessK8sWithUpload() throws Exception { - final String injectedErrMsg = "uploadFileToBookkeeper triggered"; - mockedWorkerService.getWorkerConfig().setUploadBuiltinSinksSources(true); - - mockStatic(WorkerUtils.class, ctx -> { - ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class))) - .thenThrow(new RuntimeException(injectedErrMsg)); - - }); - - NarClassLoader mockedClassLoader = mock(NarClassLoader.class); - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.getFunctionTypes(any(FunctionConfig.class), any(Class.class))).thenReturn(new Class[]{String.class, String.class}); - ctx.when(() -> FunctionCommon.convertRuntime(any(FunctionConfig.Runtime.class))).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.isFunctionCodeBuiltin(any())).thenReturn(true); - }); - - doReturn(Function.class).when(mockedClassLoader).loadClass(anyString()); - - FunctionsManager mockedFunctionsManager = mock(FunctionsManager.class); - FunctionArchive functionArchive = FunctionArchive.builder() - .classLoader(mockedClassLoader) - .build(); - when(mockedFunctionsManager.getFunction("exclamation")).thenReturn(functionArchive); - when(mockedFunctionsManager.getFunctionArchive(any())).thenReturn(getPulsarApiExamplesNar().toPath()); - - when(mockedWorkerService.getFunctionsManager()).thenReturn(mockedFunctionsManager); - - when(mockedRuntimeFactory.externallyManaged()).thenReturn(true); - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - - FunctionConfig functionConfig = createDefaultFunctionConfig(); - functionConfig.setJar("builtin://exclamation"); - - try (FileInputStream inputStream = new FileInputStream(getPulsarApiExamplesNar())) { - try { - resource.registerFunction( - tenant, - namespace, - function, - inputStream, - mockedFormData, - null, - functionConfig, - null); - Assert.fail(); - } catch (RuntimeException e) { - Assert.assertEquals(e.getMessage(), injectedErrMsg); - } - } - } - - // - // Update Functions - // - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") - public void testUpdateFunctionMissingTenant() throws Exception { - try { - testUpdateFunctionMissingArguments( - null, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "Tenant is not provided"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") - public void testUpdateFunctionMissingNamespace() throws Exception { - try { - testUpdateFunctionMissingArguments( - tenant, - null, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "Namespace is not provided"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") - public void testUpdateFunctionMissingFunctionName() throws Exception { - try { - testUpdateFunctionMissingArguments( - tenant, - namespace, - null, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "Function name is not provided"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") - public void testUpdateFunctionMissingPackage() throws Exception { - try { - mockWorkerUtils(); - testUpdateFunctionMissingArguments( - tenant, - namespace, - function, - null, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "Update contains no change"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") - public void testUpdateFunctionMissingInputTopic() throws Exception { - try { - mockWorkerUtils(); - - testUpdateFunctionMissingArguments( - tenant, - namespace, - function, - null, - null, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "Update contains no change"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") - public void testUpdateFunctionMissingClassName() throws Exception { - try { - mockWorkerUtils(); - - testUpdateFunctionMissingArguments( - tenant, - namespace, - function, - null, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - null, - parallelism, - "Update contains no change"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test - public void testUpdateFunctionChangedParallelism() throws Exception { - try { - mockWorkerUtils(); - - testUpdateFunctionMissingArguments( - tenant, - namespace, - function, - null, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - null, - parallelism + 1, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test - public void testUpdateFunctionChangedInputs() throws Exception { - mockWorkerUtils(); - - testUpdateFunctionMissingArguments( - tenant, - namespace, - function, - null, - topicsToSerDeClassName, - mockedFormData, - "DifferentOutput", - outputSerdeClassName, - null, - parallelism, - null); - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Input Topics cannot be altered") - public void testUpdateFunctionChangedOutput() throws Exception { - try { - mockWorkerUtils(); - - Map someOtherInput = new HashMap<>(); - someOtherInput.put("DifferentTopic", TopicSchema.DEFAULT_SERDE); - testUpdateFunctionMissingArguments( - tenant, - namespace, - function, - null, - someOtherInput, - mockedFormData, - outputTopic, - outputSerdeClassName, - null, - parallelism, - "Input Topics cannot be altered"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - private void testUpdateFunctionMissingArguments( - String tenant, - String namespace, - String function, - InputStream inputStream, - Map topicsToSerDeClassName, - FormDataContentDisposition details, - String outputTopic, - String outputSerdeClassName, - String className, - Integer parallelism, - String expectedError) throws Exception { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - FunctionConfig functionConfig = new FunctionConfig(); - if (tenant != null) { - functionConfig.setTenant(tenant); - } - if (namespace != null) { - functionConfig.setNamespace(namespace); - } - if (function != null) { - functionConfig.setName(function); - } - if (topicsToSerDeClassName != null) { - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - } - if (outputTopic != null) { - functionConfig.setOutput(outputTopic); - } - if (outputSerdeClassName != null) { - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - } - if (className != null) { - functionConfig.setClassName(className); - } - if (parallelism != null) { - functionConfig.setParallelism(parallelism); - } - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - - if (expectedError != null) { - doThrow(new IllegalArgumentException(expectedError)) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); - } - - resource.updateFunction( - tenant, - namespace, - function, - inputStream, - details, - null, - functionConfig, - null, null); - - } - - private void updateDefaultFunction() { - updateDefaultFunctionWithPackageUrl(null); - } - - private void updateDefaultFunctionWithPackageUrl(String packageUrl) { - FunctionConfig functionConfig = new FunctionConfig(); - functionConfig.setTenant(tenant); - functionConfig.setNamespace(namespace); - functionConfig.setName(function); - functionConfig.setClassName(className); - functionConfig.setParallelism(parallelism); - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - functionConfig.setOutput(outputTopic); - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - - resource.updateFunction( - tenant, - namespace, - function, - mockedInputStream, - mockedFormData, - packageUrl, - functionConfig, - null, null); - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist") - public void testUpdateNotExistedFunction() { - try { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - updateDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure") - public void testUpdateFunctionUploadFailure() throws Exception { - try { - mockWorkerUtils(ctx -> { - ctx.when(() -> { - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - - }).thenThrow(new IOException("upload failure")); - ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); - }); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - updateDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); - throw re; - } - } - - @Test - public void testUpdateFunctionSuccess() throws Exception { - mockWorkerUtils(); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - updateDefaultFunction(); - } - - @Test - public void testUpdateFunctionWithUrl() { - Configurator.setRootLevel(Level.DEBUG); - - String fileLocation = FutureUtil.class.getProtectionDomain().getCodeSource().getLocation().getPath(); - String filePackageUrl = "file://" + fileLocation; - - FunctionConfig functionConfig = new FunctionConfig(); - functionConfig.setOutput(outputTopic); - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - functionConfig.setTenant(tenant); - functionConfig.setNamespace(namespace); - functionConfig.setName(function); - functionConfig.setClassName(className); - functionConfig.setParallelism(parallelism); - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - resource.updateFunction( - tenant, - namespace, - function, - null, - null, - filePackageUrl, - functionConfig, - null, null); - - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to register") - public void testUpdateFunctionFailure() throws Exception { - try { - mockWorkerUtils(); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - doThrow(new IllegalArgumentException("function failed to register")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); - - updateDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registeration interrupted") - public void testUpdateFunctionInterrupted() throws Exception { - try { - mockWorkerUtils(); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - doThrow(new IllegalStateException("Function registeration interrupted")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); - - updateDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); - throw re; - } - } - - - @Test(timeOut = 20000) - public void testUpdateFunctionSuccessWithPackageName() { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - updateDefaultFunctionWithPackageUrl("function://public/default/test@v1"); - } - - @Test(timeOut = 20000) - public void testUpdateFunctionFailedWithWrongPackageName() throws PulsarAdminException { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - try { - doThrow(new PulsarAdminException("package name is invalid")) - .when(mockedPackages).download(anyString(), anyString()); - registerDefaultFunctionWithPackageUrl("function://"); - } catch (RestException e) { - // expected exception - assertEquals(e.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - } - } - - // - // deregister function - // - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") - public void testDeregisterFunctionMissingTenant() { - try { - - testDeregisterFunctionMissingArguments( - null, - namespace, - function - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") - public void testDeregisterFunctionMissingNamespace() { - try { - testDeregisterFunctionMissingArguments( - tenant, - null, - function - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") - public void testDeregisterFunctionMissingFunctionName() { - try { - testDeregisterFunctionMissingArguments( - tenant, - namespace, - null - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - private void testDeregisterFunctionMissingArguments( - String tenant, - String namespace, - String function - ) { - resource.deregisterFunction( - tenant, - namespace, - function, - null); - } - - private void deregisterDefaultFunction() { - resource.deregisterFunction( - tenant, - namespace, - function, - null); - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist") - public void testDeregisterNotExistedFunction() { - try { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - deregisterDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.NOT_FOUND); - throw re; - } - } - - @Test - public void testDeregisterFunctionSuccess() { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - deregisterDefaultFunction(); - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to deregister") - public void testDeregisterFunctionFailure() throws Exception { - try { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - doThrow(new IllegalArgumentException("function failed to deregister")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); - - deregisterDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function deregisteration interrupted") - public void testDeregisterFunctionInterrupted() throws Exception { - try { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - doThrow(new IllegalStateException("Function deregisteration interrupted")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); - - deregisterDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); - throw re; - } - } - - // - // Get Function Info - // - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") - public void testGetFunctionMissingTenant() { - try { - testGetFunctionMissingArguments( - null, - namespace, - function - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") - public void testGetFunctionMissingNamespace() { - try { - testGetFunctionMissingArguments( - tenant, - null, - function - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") - public void testGetFunctionMissingFunctionName() { - try { - testGetFunctionMissingArguments( - tenant, - namespace, - null - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - private void testGetFunctionMissingArguments( - String tenant, - String namespace, - String function - ) { - resource.getFunctionInfo( - tenant, - namespace, - function,null - ); - - } - - private FunctionConfig getDefaultFunctionInfo() { - return resource.getFunctionInfo( - tenant, - namespace, - function, - null - ); - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist") - public void testGetNotExistedFunction() { - try { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - getDefaultFunctionInfo(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.NOT_FOUND); - throw re; - } - } - - @Test - public void testGetFunctionSuccess() { - mockInstanceUtils(); - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - SinkSpec sinkSpec = SinkSpec.newBuilder() - .setTopic(outputTopic) - .setSerDeClassName(outputSerdeClassName).build(); - FunctionDetails functionDetails = FunctionDetails.newBuilder() - .setClassName(className) - .setSink(sinkSpec) - .setAutoAck(true) - .setName(function) - .setNamespace(namespace) - .setProcessingGuarantees(ProcessingGuarantees.ATMOST_ONCE) - .setTenant(tenant) - .setParallelism(parallelism) - .setSource(SourceSpec.newBuilder().setSubscriptionType(subscriptionType) - .putAllTopicsToSerDeClassName(topicsToSerDeClassName)).build(); - FunctionMetaData metaData = FunctionMetaData.newBuilder() - .setCreateTime(System.currentTimeMillis()) - .setFunctionDetails(functionDetails) - .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("/path/to/package")) - .setVersion(1234) - .build(); - when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(function))).thenReturn(metaData); - - FunctionConfig functionConfig = getDefaultFunctionInfo(); - assertEquals( - FunctionConfigUtils.convertFromDetails(functionDetails), - functionConfig); - } - - // - // List Functions - // + protected void registerFunction(String tenant, String namespace, String function, InputStream inputStream, + FormDataContentDisposition details, String functionPkgUrl, FunctionConfig functionConfig) { + resource.registerFunction( + tenant, + namespace, + function, + inputStream, + details, + functionPkgUrl, + functionConfig, + null); + } + protected void updateFunction(String tenant, + String namespace, + String functionName, + InputStream uploadedInputStream, + FormDataContentDisposition fileDetail, + String functionPkgUrl, + FunctionConfig functionConfig, + AuthenticationParameters authParams, + UpdateOptionsImpl updateOptions) { + resource.updateFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail, functionPkgUrl, + functionConfig, authParams, updateOptions); + } - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") - public void testListFunctionsMissingTenant() { - try { - testListFunctionsMissingArguments( - null, - namespace - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } + protected StreamingOutput downloadFunction(String tenant, String namespace, String componentName, + AuthenticationParameters authParams, boolean transformFunction) { + return resource.downloadFunction(tenant, namespace, componentName, authParams, transformFunction); } - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") - public void testListFunctionsMissingNamespace() { - try { - testListFunctionsMissingArguments( - tenant, - null - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; + protected File downloadFunction(final String path, final AuthenticationParameters authParams) throws IOException { + StreamingOutput streamingOutput = resource.downloadFunction(path, authParams); + File pkgFile = File.createTempFile("testpkg", "nar"); + try(OutputStream output = new FileOutputStream(pkgFile)) { + streamingOutput.write(output); } + return pkgFile; } - private void testListFunctionsMissingArguments( + protected void testDeregisterFunctionMissingArguments( String tenant, - String namespace + String namespace, + String function ) { - resource.listFunctions( - tenant, - namespace,null - ); - - } - - private List listDefaultFunctions() { - return resource.listFunctions( - tenant, - namespace,null - ); - } - - @Test - public void testListFunctionsSuccess() { - mockInstanceUtils(); - final List functions = Lists.newArrayList("test-1", "test-2"); - final List metaDataList = new LinkedList<>(); - FunctionMetaData functionMetaData1 = FunctionMetaData.newBuilder().setFunctionDetails( - FunctionDetails.newBuilder().setName("test-1").build() - ).build(); - FunctionMetaData functionMetaData2 = FunctionMetaData.newBuilder().setFunctionDetails( - FunctionDetails.newBuilder().setName("test-2").build() - ).build(); - metaDataList.add(functionMetaData1); - metaDataList.add(functionMetaData2); - when(mockedManager.listFunctions(eq(tenant), eq(namespace))).thenReturn(metaDataList); - - List functionList = listDefaultFunctions(); - assertEquals(functions, functionList); + resource.deregisterFunction( + tenant, + namespace, + function, + null); } - @Test - public void testOnlyGetSources() { - List functions = Lists.newArrayList("test-2"); - List functionMetaDataList = new LinkedList<>(); - FunctionMetaData f1 = FunctionMetaData.newBuilder().setFunctionDetails( - FunctionDetails.newBuilder() - .setName("test-1") - .setComponentType(FunctionDetails.ComponentType.SOURCE) - .build()).build(); - functionMetaDataList.add(f1); - FunctionMetaData f2 = FunctionMetaData.newBuilder().setFunctionDetails( - FunctionDetails.newBuilder() - .setName("test-2") - .setComponentType(FunctionDetails.ComponentType.FUNCTION) - .build()).build(); - functionMetaDataList.add(f2); - FunctionMetaData f3 = FunctionMetaData.newBuilder().setFunctionDetails( - FunctionDetails.newBuilder() - .setName("test-3") - .setComponentType(FunctionDetails.ComponentType.SINK) - .build()).build(); - functionMetaDataList.add(f3); - when(mockedManager.listFunctions(eq(tenant), eq(namespace))).thenReturn(functionMetaDataList); - - List functionList = listDefaultFunctions(); - assertEquals(functions, functionList); + protected void deregisterDefaultFunction() { + resource.deregisterFunction( + tenant, + namespace, + function, + null); } - @Test - public void testDownloadFunctionHttpUrl() throws Exception { - String jarHttpUrl = - "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar"; - String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); + protected void testGetFunctionMissingArguments( + String tenant, + String namespace, + String function + ) { + resource.getFunctionInfo( + tenant, + namespace, + function, null + ); - StreamingOutput streamOutput = resource.downloadFunction(jarHttpUrl, null); - File pkgFile = new File(testDir, UUID.randomUUID().toString()); - OutputStream output = new FileOutputStream(pkgFile); - streamOutput.write(output); - Assert.assertTrue(pkgFile.exists()); - pkgFile.delete(); } - @Test - public void testDownloadFunctionFile() throws Exception { - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath().replace('\\', '/'); - String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); - - StreamingOutput streamOutput = resource.downloadFunction("file:///" + fileLocation, null); - File pkgFile = new File(testDir, UUID.randomUUID().toString()); - OutputStream output = new FileOutputStream(pkgFile); - streamOutput.write(output); - Assert.assertTrue(pkgFile.exists()); - Assert.assertEquals(file.length(), pkgFile.length()); - pkgFile.delete(); + protected FunctionConfig getDefaultFunctionInfo() { + return resource.getFunctionInfo( + tenant, + namespace, + function, + null + ); } - @Test - public void testDownloadFunctionBuiltinConnector() throws Exception { - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); - - WorkerConfig config = new WorkerConfig() - .setUploadBuiltinSinksSources(false); - when(mockedWorkerService.getWorkerConfig()).thenReturn(config); - - Connector connector = Connector.builder().archivePath(file.toPath()).build(); - ConnectorsManager connectorsManager = mock(ConnectorsManager.class); - when(connectorsManager.getConnector("cassandra")).thenReturn(connector); - when(mockedWorkerService.getConnectorsManager()).thenReturn(connectorsManager); - - StreamingOutput streamOutput = resource.downloadFunction("builtin://cassandra", null); + protected void testListFunctionsMissingArguments( + String tenant, + String namespace + ) { + resource.listFunctions( + tenant, + namespace, null + ); - File pkgFile = new File(testDir, UUID.randomUUID().toString()); - OutputStream output = new FileOutputStream(pkgFile); - streamOutput.write(output); - output.flush(); - output.close(); - Assert.assertTrue(pkgFile.exists()); - Assert.assertTrue(pkgFile.exists()); - Assert.assertEquals(file.length(), pkgFile.length()); - pkgFile.delete(); } - @Test - public void testDownloadFunctionBuiltinFunction() throws Exception { - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); - - WorkerConfig config = new WorkerConfig() - .setUploadBuiltinSinksSources(false); - when(mockedWorkerService.getWorkerConfig()).thenReturn(config); - - FunctionsManager functionsManager = mock(FunctionsManager.class); - FunctionArchive functionArchive = FunctionArchive.builder().archivePath(file.toPath()).build(); - when(functionsManager.getFunction("exclamation")).thenReturn(functionArchive); - when(mockedWorkerService.getConnectorsManager()).thenReturn(mock(ConnectorsManager.class)); - when(mockedWorkerService.getFunctionsManager()).thenReturn(functionsManager); - - StreamingOutput streamOutput = resource.downloadFunction("builtin://exclamation", null); - - File pkgFile = new File(testDir, UUID.randomUUID().toString()); - OutputStream output = new FileOutputStream(pkgFile); - streamOutput.write(output); - output.flush(); - output.close(); - Assert.assertTrue(pkgFile.exists()); - Assert.assertEquals(file.length(), pkgFile.length()); - pkgFile.delete(); + protected List listDefaultFunctions() { + return resource.listFunctions( + tenant, + namespace, null + ); } @Test public void testDownloadFunctionBuiltinConnectorByName() throws Exception { - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); + File file = getPulsarApiExamplesNar(); WorkerConfig config = new WorkerConfig() - .setUploadBuiltinSinksSources(false); + .setUploadBuiltinSinksSources(false); when(mockedWorkerService.getWorkerConfig()).thenReturn(config); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - FunctionMetaData metaData = FunctionMetaData.newBuilder() - .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("builtin://cassandra")) - .setTransformFunctionPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("http://invalid")) - .setFunctionDetails(FunctionDetails.newBuilder().setComponentType(FunctionDetails.ComponentType.SINK)) + Function.FunctionMetaData metaData = Function.FunctionMetaData.newBuilder() + .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("builtin://cassandra")) + .setTransformFunctionPackageLocation( + Function.PackageLocationMetaData.newBuilder().setPackagePath("http://invalid")) + .setFunctionDetails(Function.FunctionDetails.newBuilder().setComponentType(Function.FunctionDetails.ComponentType.SINK)) .build(); when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(function))).thenReturn(metaData); - Connector connector = Connector.builder().archivePath(file.toPath()).build(); - ConnectorsManager connectorsManager = mock(ConnectorsManager.class); - when(connectorsManager.getConnector("cassandra")).thenReturn(connector); - when(mockedWorkerService.getConnectorsManager()).thenReturn(connectorsManager); + registerBuiltinConnector("cassandra", file); - StreamingOutput streamOutput = resource.downloadFunction(tenant, namespace, function, + StreamingOutput streamOutput = downloadFunction(tenant, namespace, function, AuthenticationParameters.builder().build(), false); - File pkgFile = new File(testDir, UUID.randomUUID().toString()); + File pkgFile = File.createTempFile("testpkg", "nar"); OutputStream output = new FileOutputStream(pkgFile); streamOutput.write(output); Assert.assertTrue(pkgFile.exists()); @@ -1812,31 +189,27 @@ public void testDownloadFunctionBuiltinConnectorByName() throws Exception { @Test public void testDownloadFunctionBuiltinFunctionByName() throws Exception { - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); + File file = getPulsarApiExamplesNar(); WorkerConfig config = new WorkerConfig() - .setUploadBuiltinSinksSources(false); + .setUploadBuiltinSinksSources(false); when(mockedWorkerService.getWorkerConfig()).thenReturn(config); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - FunctionMetaData metaData = FunctionMetaData.newBuilder() - .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("builtin://exclamation")) - .setTransformFunctionPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("http://invalid")) - .setFunctionDetails(FunctionDetails.newBuilder().setComponentType(FunctionDetails.ComponentType.FUNCTION)) - .build(); + Function.FunctionMetaData metaData = Function.FunctionMetaData.newBuilder() + .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("builtin://exclamation")) + .setTransformFunctionPackageLocation( + Function.PackageLocationMetaData.newBuilder().setPackagePath("http://invalid")) + .setFunctionDetails( + Function.FunctionDetails.newBuilder().setComponentType(Function.FunctionDetails.ComponentType.FUNCTION)) + .build(); when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(function))).thenReturn(metaData); - FunctionsManager functionsManager = mock(FunctionsManager.class); - FunctionArchive functionArchive = FunctionArchive.builder().archivePath(file.toPath()).build(); - when(functionsManager.getFunction("exclamation")).thenReturn(functionArchive); - when(mockedWorkerService.getConnectorsManager()).thenReturn(mock(ConnectorsManager.class)); - when(mockedWorkerService.getFunctionsManager()).thenReturn(functionsManager); + registerBuiltinFunction("exclamation", file); - StreamingOutput streamOutput = resource.downloadFunction(tenant, namespace, function, + StreamingOutput streamOutput = downloadFunction(tenant, namespace, function, AuthenticationParameters.builder().build(), false); - File pkgFile = new File(testDir, UUID.randomUUID().toString()); + File pkgFile = File.createTempFile("testpkg", "nar"); OutputStream output = new FileOutputStream(pkgFile); streamOutput.write(output); Assert.assertTrue(pkgFile.exists()); @@ -1846,32 +219,26 @@ public void testDownloadFunctionBuiltinFunctionByName() throws Exception { @Test public void testDownloadTransformFunctionByName() throws Exception { - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); + File file = getPulsarApiExamplesNar(); WorkerConfig workerConfig = new WorkerConfig() - .setUploadBuiltinSinksSources(false); + .setUploadBuiltinSinksSources(false); when(mockedWorkerService.getWorkerConfig()).thenReturn(workerConfig); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - FunctionMetaData metaData = FunctionMetaData.newBuilder() - .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("http://invalid")) - .setTransformFunctionPackageLocation(PackageLocationMetaData.newBuilder() + Function.FunctionMetaData metaData = Function.FunctionMetaData.newBuilder() + .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("http://invalid")) + .setTransformFunctionPackageLocation(Function.PackageLocationMetaData.newBuilder() .setPackagePath("builtin://exclamation")) .build(); when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(function))).thenReturn(metaData); - FunctionsManager functionsManager = mock(FunctionsManager.class); - FunctionArchive functionArchive = FunctionArchive.builder().archivePath(file.toPath()).build(); - when(functionsManager.getFunction("exclamation")).thenReturn(functionArchive); - when(mockedWorkerService.getConnectorsManager()).thenReturn(mock(ConnectorsManager.class)); - when(mockedWorkerService.getFunctionsManager()).thenReturn(functionsManager); + registerBuiltinFunction("exclamation", file); - StreamingOutput streamOutput = resource.downloadFunction(tenant, namespace, function, + StreamingOutput streamOutput = downloadFunction(tenant, namespace, function, AuthenticationParameters.builder().build(), true); - File pkgFile = new File(testDir, UUID.randomUUID().toString()); + File pkgFile = File.createTempFile("testpkg", "nar"); OutputStream output = new FileOutputStream(pkgFile); streamOutput.write(output); Assert.assertTrue(pkgFile.exists()); @@ -1879,15 +246,58 @@ public void testDownloadTransformFunctionByName() throws Exception { pkgFile.delete(); } + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't" + + " exist") + public void testGetNotExistedFunction() throws IOException { + try { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); + getDefaultFunctionInfo(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.NOT_FOUND); + throw re; + } + } @Test - public void testRegisterFunctionFileUrlWithValidSinkClass() throws Exception { + public void testGetFunctionSuccess() throws IOException { + mockInstanceUtils(); + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + Function.SinkSpec sinkSpec = Function.SinkSpec.newBuilder() + .setTopic(outputTopic) + .setSerDeClassName(outputSerdeClassName).build(); + Function.FunctionDetails functionDetails = Function.FunctionDetails.newBuilder() + .setClassName(className) + .setSink(sinkSpec) + .setAutoAck(true) + .setName(function) + .setNamespace(namespace) + .setProcessingGuarantees(Function.ProcessingGuarantees.ATMOST_ONCE) + .setTenant(tenant) + .setParallelism(parallelism) + .setSource(Function.SourceSpec.newBuilder().setSubscriptionType(subscriptionType) + .putAllTopicsToSerDeClassName(topicsToSerDeClassName)).build(); + Function.FunctionMetaData metaData = Function.FunctionMetaData.newBuilder() + .setCreateTime(System.currentTimeMillis()) + .setFunctionDetails(functionDetails) + .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("/path/to/package")) + .setVersion(1234) + .build(); + when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(function))).thenReturn(metaData); + + FunctionConfig functionConfig = getDefaultFunctionInfo(); + assertEquals( + FunctionConfigUtils.convertFromDetails(functionDetails), + functionConfig); + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function language runtime is " + + "either not set or cannot be determined") + public void testCreateFunctionWithoutSettingRuntime() throws Exception { Configurator.setRootLevel(Level.DEBUG); - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath().replace('\\', '/'); - String filePackageUrl = "file:///" + fileLocation; + File file = getPulsarApiExamplesNar(); + String filePackageUrl = file.toURI().toString(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); FunctionConfig functionConfig = new FunctionConfig(); @@ -1896,82 +306,135 @@ public void testRegisterFunctionFileUrlWithValidSinkClass() throws Exception { functionConfig.setName(function); functionConfig.setClassName(className); functionConfig.setParallelism(parallelism); - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); functionConfig.setOutput(outputTopic); functionConfig.setOutputSerdeClassName(outputSerdeClassName); - resource.registerFunction(tenant, namespace, function, null, null, filePackageUrl, functionConfig, null); + registerFunction(tenant, namespace, function, null, null, filePackageUrl, functionConfig); } @Test - public void testRegisterFunctionWithConflictingFields() throws Exception { - Configurator.setRootLevel(Level.DEBUG); - String actualTenant = "DIFFERENT_TENANT"; - String actualNamespace = "DIFFERENT_NAMESPACE"; - String actualName = "DIFFERENT_NAME"; - this.namespaceList.add(actualTenant + "/" + actualNamespace); + public void testUpdateSourceWithNoChange() throws IOException { + mockWorkerUtils(); - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath().replace('\\', '/'); - String filePackageUrl = "file:///" + fileLocation; when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - when(mockedManager.containsFunction(eq(actualTenant), eq(actualNamespace), eq(actualName))).thenReturn(false); + FunctionConfig funcConfig = createDefaultFunctionConfig(); - FunctionConfig functionConfig = new FunctionConfig(); - functionConfig.setTenant(tenant); - functionConfig.setNamespace(namespace); - functionConfig.setName(function); - functionConfig.setClassName(className); - functionConfig.setParallelism(parallelism); - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - functionConfig.setOutput(outputTopic); - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - resource.registerFunction(actualTenant, actualNamespace, actualName, null, null, filePackageUrl, functionConfig, - null); + // config has not changes and don't update auth, should fail + try { + updateFunction( + funcConfig.getTenant(), + funcConfig.getNamespace(), + funcConfig.getName(), + null, + mockedFormData, + null, + funcConfig, + null, + null); + fail("Update without changes should fail"); + } catch (RestException e) { + assertThat(e.getMessage()).contains("Update contains no change"); + } + + try { + UpdateOptionsImpl updateOptions = new UpdateOptionsImpl(); + updateOptions.setUpdateAuthData(false); + updateFunction( + funcConfig.getTenant(), + funcConfig.getNamespace(), + funcConfig.getName(), + null, + mockedFormData, + null, + funcConfig, + null, + updateOptions); + fail("Update without changes should fail"); + } catch (RestException e) { + assertTrue(e.getMessage().contains("Update contains no change")); + } + + // no changes but set the auth-update flag to true, should not fail + UpdateOptionsImpl updateOptions = new UpdateOptionsImpl(); + updateOptions.setUpdateAuthData(true); + updateFunction( + funcConfig.getTenant(), + funcConfig.getNamespace(), + funcConfig.getName(), + null, + mockedFormData, + null, + funcConfig, + null, + updateOptions); } - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function language runtime is either not set or cannot be determined") - public void testCreateFunctionWithoutSettingRuntime() throws Exception { - Configurator.setRootLevel(Level.DEBUG); + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function config is not provided") + public void testMissingFunctionConfig() throws IOException { + registerFunction(tenant, namespace, function, mockedInputStream, mockedFormData, null, null); + } + + /* + Externally managed runtime, + uploadBuiltinSinksSources == false + Make sure uploadFileToBookkeeper is not called + */ + @Test + public void testRegisterFunctionSuccessK8sNoUpload() throws Exception { + mockedWorkerService.getWorkerConfig().setUploadBuiltinSinksSources(false); + + mockStatic(WorkerUtils.class, ctx -> { + ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( + anyString(), + any(File.class), + any(Namespace.class))) + .thenThrow(new RuntimeException("uploadFileToBookkeeper triggered")); + + }); - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath().replace('\\', '/'); - String filePackageUrl = "file:///" + fileLocation; + registerBuiltinFunction("exclamation", getPulsarApiExamplesNar()); + when(mockedRuntimeFactory.externallyManaged()).thenReturn(true); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - FunctionConfig functionConfig = new FunctionConfig(); - functionConfig.setTenant(tenant); - functionConfig.setNamespace(namespace); - functionConfig.setName(function); - functionConfig.setClassName(className); - functionConfig.setParallelism(parallelism); - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - functionConfig.setOutput(outputTopic); - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - resource.registerFunction(tenant, namespace, function, null, null, filePackageUrl, functionConfig, null); + FunctionConfig functionConfig = createDefaultFunctionConfig(); + functionConfig.setJar("builtin://exclamation"); + registerFunction(tenant, namespace, function, null, mockedFormData, null, functionConfig); } - public static FunctionConfig createDefaultFunctionConfig() { - FunctionConfig functionConfig = new FunctionConfig(); - functionConfig.setTenant(tenant); - functionConfig.setNamespace(namespace); - functionConfig.setName(function); - functionConfig.setClassName(className); - functionConfig.setParallelism(parallelism); - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - functionConfig.setOutput(outputTopic); - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - return functionConfig; - } + /* + Externally managed runtime, + uploadBuiltinSinksSources == true + Make sure uploadFileToBookkeeper is called + */ + @Test + public void testRegisterFunctionSuccessK8sWithUpload() throws Exception { + final String injectedErrMsg = "uploadFileToBookkeeper triggered"; + mockedWorkerService.getWorkerConfig().setUploadBuiltinSinksSources(true); + + mockStatic(WorkerUtils.class, ctx -> { + ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( + anyString(), + any(File.class), + any(Namespace.class))) + .thenThrow(new RuntimeException(injectedErrMsg)); + + }); + + registerBuiltinFunction("exclamation", getPulsarApiExamplesNar()); + when(mockedRuntimeFactory.externallyManaged()).thenReturn(true); + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - public static FunctionDetails createDefaultFunctionDetails() { FunctionConfig functionConfig = createDefaultFunctionConfig(); - return FunctionConfigUtils.convert(functionConfig, (ClassLoader) null); + functionConfig.setJar("builtin://exclamation"); + + try { + registerFunction(tenant, namespace, function, null, mockedFormData, null, functionConfig); + Assert.fail(); + } catch (RuntimeException e) { + Assert.assertEquals(e.getMessage(), injectedErrMsg); + } } + } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java index 5dcc795304ef5..c6c6303e48007 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java @@ -18,246 +18,86 @@ */ package org.apache.pulsar.functions.worker.rest.api.v3; +import static org.apache.pulsar.functions.proto.Function.ProcessingGuarantees.ATLEAST_ONCE; +import static org.apache.pulsar.functions.source.TopicSchema.DEFAULT_SERDE; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import com.google.common.collect.Lists; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; +import java.lang.reflect.Method; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.function.Consumer; import javax.ws.rs.core.Response; import org.apache.distributedlog.api.namespace.Namespace; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; import org.apache.pulsar.broker.authentication.AuthenticationParameters; -import org.apache.pulsar.client.admin.Functions; -import org.apache.pulsar.client.admin.Namespaces; -import org.apache.pulsar.client.admin.Packages; -import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.admin.Tenants; -import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.functions.UpdateOptionsImpl; import org.apache.pulsar.common.functions.Utils; import org.apache.pulsar.common.io.SinkConfig; -import org.apache.pulsar.common.nar.NarClassLoader; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; -import org.apache.pulsar.common.util.ClassLoaderUtils; import org.apache.pulsar.common.util.RestException; -import org.apache.pulsar.functions.api.examples.ExclamationFunction; import org.apache.pulsar.functions.api.examples.RecordFunction; import org.apache.pulsar.functions.api.utils.IdentityFunction; import org.apache.pulsar.functions.instance.InstanceUtils; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.Function.FunctionDetails; import org.apache.pulsar.functions.proto.Function.FunctionMetaData; -import org.apache.pulsar.functions.runtime.RuntimeFactory; -import org.apache.pulsar.functions.utils.FunctionCommon; import org.apache.pulsar.functions.utils.SinkConfigUtils; -import org.apache.pulsar.functions.utils.functions.FunctionArchive; -import org.apache.pulsar.functions.utils.functions.FunctionUtils; -import org.apache.pulsar.functions.utils.io.Connector; -import org.apache.pulsar.functions.utils.io.ConnectorUtils; -import org.apache.pulsar.functions.worker.ConnectorsManager; -import org.apache.pulsar.functions.worker.FunctionMetaDataManager; -import org.apache.pulsar.functions.worker.FunctionRuntimeManager; -import org.apache.pulsar.functions.worker.FunctionsManager; -import org.apache.pulsar.functions.worker.LeaderService; -import org.apache.pulsar.functions.worker.PulsarWorkerService; import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerUtils; -import org.apache.pulsar.functions.worker.rest.api.PulsarFunctionTestTemporaryDirectory; import org.apache.pulsar.functions.worker.rest.api.SinksImpl; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.mockito.MockedStatic; import org.mockito.Mockito; import org.testng.Assert; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import static org.apache.pulsar.functions.proto.Function.ProcessingGuarantees.ATLEAST_ONCE; -import static org.apache.pulsar.functions.source.TopicSchema.DEFAULT_SERDE; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.anyString; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; /** * Unit test of {@link SinksApiV3Resource}. */ -public class SinkApiV3ResourceTest { +public class SinkApiV3ResourceTest extends AbstractFunctionsResourceTest { - private static final String tenant = "test-tenant"; - private static final String namespace = "test-namespace"; private static final String sink = "test-sink"; - private static final Map topicsToSerDeClassName = new HashMap<>(); - - static { - topicsToSerDeClassName.put("test_src", DEFAULT_SERDE); - } - - private static final String subscriptionName = "test-subscription"; - private static final String CASSANDRA_STRING_SINK = "org.apache.pulsar.io.cassandra.CassandraStringSink"; - private static final int parallelism = 1; - - private PulsarWorkerService mockedWorkerService; - private PulsarAdmin mockedPulsarAdmin; - private Tenants mockedTenants; - private Namespaces mockedNamespaces; - private Functions mockedFunctions; - private TenantInfoImpl mockedTenantInfo; - private List namespaceList = new LinkedList<>(); - private FunctionMetaDataManager mockedManager; - private FunctionRuntimeManager mockedFunctionRunTimeManager; - private RuntimeFactory mockedRuntimeFactory; - private Namespace mockedNamespace; + private SinksImpl resource; - private InputStream mockedInputStream; - private FormDataContentDisposition mockedFormData; - private FunctionMetaData mockedFunctionMetaData; - private LeaderService mockedLeaderService; - private Packages mockedPackages; - private PulsarFunctionTestTemporaryDirectory tempDirectory; - private static Map mockStaticContexts = new HashMap<>(); - - private static final String SYSTEM_PROPERTY_NAME_CASSANDRA_NAR_FILE_PATH = "pulsar-io-cassandra.nar.path"; - - public static File getPulsarIOCassandraNar() { - return new File(Objects.requireNonNull(System.getProperty(SYSTEM_PROPERTY_NAME_CASSANDRA_NAR_FILE_PATH) - , "pulsar-io-cassandra.nar file location must be specified with " - + SYSTEM_PROPERTY_NAME_CASSANDRA_NAR_FILE_PATH + " system property")); - } - - private static final String SYSTEM_PROPERTY_NAME_TWITTER_NAR_FILE_PATH = "pulsar-io-twitter.nar.path"; - - public static File getPulsarIOTwitterNar() { - return new File(Objects.requireNonNull(System.getProperty(SYSTEM_PROPERTY_NAME_TWITTER_NAR_FILE_PATH) - , "pulsar-io-twitter.nar file location must be specified with " - + SYSTEM_PROPERTY_NAME_TWITTER_NAR_FILE_PATH + " system property")); - } - - private static final String SYSTEM_PROPERTY_NAME_INVALID_NAR_FILE_PATH = "pulsar-io-invalid.nar.path"; - - public static File getPulsarIOInvalidNar() { - return new File(Objects.requireNonNull(System.getProperty(SYSTEM_PROPERTY_NAME_INVALID_NAR_FILE_PATH) - , "invalid nar file location must be specified with " - + SYSTEM_PROPERTY_NAME_INVALID_NAR_FILE_PATH + " system property")); - } - - @BeforeMethod - public void setup() throws Exception { - this.mockedManager = mock(FunctionMetaDataManager.class); - this.mockedFunctionRunTimeManager = mock(FunctionRuntimeManager.class); - this.mockedRuntimeFactory = mock(RuntimeFactory.class); - this.mockedInputStream = mock(InputStream.class); - this.mockedNamespace = mock(Namespace.class); - this.mockedFormData = mock(FormDataContentDisposition.class); - when(mockedFormData.getFileName()).thenReturn("test"); - this.mockedTenantInfo = mock(TenantInfoImpl.class); - this.mockedPulsarAdmin = mock(PulsarAdmin.class); - this.mockedTenants = mock(Tenants.class); - this.mockedNamespaces = mock(Namespaces.class); - this.mockedFunctions = mock(Functions.class); - this.mockedLeaderService = mock(LeaderService.class); - this.mockedPackages = mock(Packages.class); - namespaceList.add(tenant + "/" + namespace); - - this.mockedWorkerService = mock(PulsarWorkerService.class); - when(mockedWorkerService.getFunctionMetaDataManager()).thenReturn(mockedManager); - when(mockedWorkerService.getLeaderService()).thenReturn(mockedLeaderService); - when(mockedWorkerService.getFunctionRuntimeManager()).thenReturn(mockedFunctionRunTimeManager); - when(mockedFunctionRunTimeManager.getRuntimeFactory()).thenReturn(mockedRuntimeFactory); - when(mockedWorkerService.getDlogNamespace()).thenReturn(mockedNamespace); - when(mockedWorkerService.isInitialized()).thenReturn(true); - when(mockedWorkerService.getBrokerAdmin()).thenReturn(mockedPulsarAdmin); - when(mockedWorkerService.getFunctionAdmin()).thenReturn(mockedPulsarAdmin); - when(mockedPulsarAdmin.tenants()).thenReturn(mockedTenants); - when(mockedPulsarAdmin.namespaces()).thenReturn(mockedNamespaces); - when(mockedPulsarAdmin.functions()).thenReturn(mockedFunctions); - when(mockedPulsarAdmin.packages()).thenReturn(mockedPackages); - when(mockedTenants.getTenantInfo(any())).thenReturn(mockedTenantInfo); - when(mockedNamespaces.getNamespaces(any())).thenReturn(namespaceList); - when(mockedLeaderService.isLeader()).thenReturn(true); - doAnswer(invocationOnMock -> { - Files.copy(getPulsarIOCassandraNar().toPath(), Paths.get(invocationOnMock.getArgument(1, String.class)), - StandardCopyOption.REPLACE_EXISTING); - return null; - }).when(mockedPackages).download(any(), any()); - - // worker config - WorkerConfig workerConfig = new WorkerConfig() - .setWorkerId("test") - .setWorkerPort(8080) - .setFunctionMetadataTopicName("pulsar/functions") - .setNumFunctionPackageReplicas(3) - .setPulsarServiceUrl("pulsar://localhost:6650/"); - tempDirectory = PulsarFunctionTestTemporaryDirectory.create(getClass().getSimpleName()); - tempDirectory.useTemporaryDirectoriesForWorkerConfig(workerConfig); - when(mockedWorkerService.getWorkerConfig()).thenReturn(workerConfig); + @Override + protected void doSetup() { this.resource = spy(new SinksImpl(() -> mockedWorkerService)); - } - @AfterMethod(alwaysRun = true) - public void cleanup() { - if (tempDirectory != null) { - tempDirectory.delete(); + @Override + protected void customizeWorkerConfig(WorkerConfig workerConfig, Method method) { + if (method.getName().contains("Upload") || method.getName().contains("BKPackage")) { + workerConfig.setFunctionsWorkerEnablePackageManagement(false); } - mockStaticContexts.values().forEach(MockedStatic::close); - mockStaticContexts.clear(); } - - private void mockStatic(Class classStatic, Consumer> consumer) { - final MockedStatic mockedStatic = - mockStaticContexts.computeIfAbsent(classStatic.getName(), name -> Mockito.mockStatic(classStatic)); - consumer.accept(mockedStatic); - } - - private void mockWorkerUtils() { - mockWorkerUtils(null); - } - - private void mockWorkerUtils(Consumer> consumer) { - mockStatic(WorkerUtils.class, ctx -> { - ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); - if (consumer != null) { - consumer.accept(ctx); - } - }); + @Override + protected Function.FunctionDetails.ComponentType getComponentType() { + return Function.FunctionDetails.ComponentType.SINK; } - private void mockInstanceUtils() { - mockStatic(InstanceUtils.class, ctx -> { - ctx.when(() -> InstanceUtils.calculateSubjectType(any())) - .thenReturn(FunctionDetails.ComponentType.SINK); - }); + @Override + protected File getDefaultNarFile() { + return getPulsarIOCassandraNar(); } - - // - // Register Functions - // - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") public void testRegisterSinkMissingTenant() { try { @@ -337,8 +177,8 @@ public void testRegisterSinkMissingPackage() { } } - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Sink class UnknownClass must " - + "be in class path") + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Sink class UnknownClass not " + + "found") public void testRegisterSinkWrongClassName() { mockInstanceUtils(); try { @@ -359,10 +199,8 @@ public void testRegisterSinkWrongClassName() { } } - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Sink package does not have the" - + " correct format. Pulsar cannot determine if the package is a NAR package" - + " or JAR package. Sink classname is not provided and attempts to load it as a NAR package produced the " - + "following error.") + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Sink package doesn't contain " + + "the META-INF/services/pulsar-io.yaml file.") public void testRegisterSinkMissingPackageDetails() { mockInstanceUtils(); try { @@ -722,30 +560,11 @@ public void testRegisterSinkSuccessWithTransformFunction() throws Exception { when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(false); - NarClassLoader mockedClassLoader = mock(NarClassLoader.class); - doReturn(RecordFunction.class).when(mockedClassLoader).loadClass("RecordFunction"); - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(FunctionCommon::createPkgTempFile).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getRawFunctionTypes(any(), anyBoolean())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getFunctionTypes(any(), anyBoolean())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getFunctionClassParent(any(), anyBoolean())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mockedClassLoader); - }); - - mockStatic(FunctionUtils.class, ctx -> { - ctx.when(() -> FunctionUtils.getFunctionClass(any())).thenReturn("RecordFunction"); - }); - - FunctionsManager mockedFunctionsManager = mock(FunctionsManager.class); - FunctionArchive functionArchive = FunctionArchive.builder() - .classLoader(mockedClassLoader) - .build(); - when(mockedFunctionsManager.getFunction("transform")).thenReturn(functionArchive); - - when(mockedWorkerService.getFunctionsManager()).thenReturn(mockedFunctionsManager); + registerBuiltinConnector("recordfunction", RecordFunction.class.getName()); + registerBuiltinFunction("transform", RecordFunction.class.getName()); SinkConfig sinkConfig = createDefaultSinkConfig(); + sinkConfig.setSinkType("builtin://recordfunction"); sinkConfig.setTransformFunction("builtin://transform"); sinkConfig.setTransformFunctionConfig("{\"dummy\": \"dummy\"}"); @@ -770,28 +589,7 @@ public void testRegisterSinkFailureWithInvalidTransformFunction() throws Excepti when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(false); - NarClassLoader mockedClassLoader = mock(NarClassLoader.class); - doReturn(ExclamationFunction.class).when(mockedClassLoader).loadClass("ExclamationFunction"); - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(FunctionCommon::createPkgTempFile).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getRawFunctionTypes(any(), anyBoolean())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getFunctionTypes(any(), anyBoolean())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getFunctionClassParent(any(), anyBoolean())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mockedClassLoader); - }); - - mockStatic(FunctionUtils.class, ctx -> { - ctx.when(() -> FunctionUtils.getFunctionClass(any())).thenReturn("ExclamationFunction"); - }); - - FunctionsManager mockedFunctionsManager = mock(FunctionsManager.class); - FunctionArchive functionArchive = FunctionArchive.builder() - .classLoader(mockedClassLoader) - .build(); - when(mockedFunctionsManager.getFunction("transform")).thenReturn(functionArchive); - - when(mockedWorkerService.getFunctionsManager()).thenReturn(mockedFunctionsManager); + registerBuiltinFunction("transform", getPulsarApiExamplesNar()); SinkConfig sinkConfig = createDefaultSinkConfig(); sinkConfig.setTransformFunction("builtin://transform"); @@ -946,16 +744,18 @@ public void testUpdateSinkDifferentInputs() throws Exception { public void testUpdateSinkDifferentParallelism() throws Exception { mockWorkerUtils(); - testUpdateSinkMissingArguments( - tenant, - namespace, - sink, - null, - mockedFormData, - topicsToSerDeClassName, - CASSANDRA_STRING_SINK, - parallelism + 1, - null); + try (FileInputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) { + testUpdateSinkMissingArguments( + tenant, + namespace, + sink, + inputStream, + mockedFormData, + topicsToSerDeClassName, + CASSANDRA_STRING_SINK, + parallelism + 1, + null); + } } private void testUpdateSinkMissingArguments( @@ -1007,32 +807,6 @@ private void testUpdateSinkMissingArguments( } - private void mockFunctionCommon(String tenant, String namespace, String sink) throws IOException { - mockStatic(ConnectorUtils.class, ctx -> { - ctx.when(() -> ConnectorUtils.getIOSinkClass(any(NarClassLoader.class))) - .thenReturn(CASSANDRA_STRING_SINK); - }); - - mockStatic(ClassLoaderUtils.class, ctx -> { - }); - - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.createPkgTempFile()).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getSinkType(any())).thenReturn(String.class); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mock(NarClassLoader.class)); - ctx.when(() -> FunctionCommon - .convertProcessingGuarantee(eq(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE))) - .thenReturn(ATLEAST_ONCE); - }); - - this.mockedFunctionMetaData = - FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); - when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))).thenReturn(mockedFunctionMetaData); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); - } - private void updateDefaultSink() throws Exception { updateDefaultSinkWithPackageUrl(null); } @@ -1040,25 +814,6 @@ private void updateDefaultSink() throws Exception { private void updateDefaultSinkWithPackageUrl(String packageUrl) throws Exception { SinkConfig sinkConfig = createDefaultSinkConfig(); - mockStatic(ConnectorUtils.class, ctx -> { - ctx.when(() -> ConnectorUtils.getIOSinkClass(any(NarClassLoader.class))) - .thenReturn(CASSANDRA_STRING_SINK); - }); - - mockStatic(ClassLoaderUtils.class, ctx -> { - }); - - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.createPkgTempFile()).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getSinkType(any())).thenReturn(String.class); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mock(NarClassLoader.class)); - ctx.when(() -> FunctionCommon - .convertProcessingGuarantee(eq(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE))) - .thenReturn(ATLEAST_ONCE); - }); - - this.mockedFunctionMetaData = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetaData); @@ -1091,7 +846,6 @@ public void testUpdateNotExistedSink() throws Exception { public void testUpdateSinkUploadFailure() throws Exception { try { mockWorkerUtils(ctx -> { - ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( anyString(), any(File.class), @@ -1127,24 +881,6 @@ public void testUpdateSinkWithUrl() throws Exception { when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); - mockStatic(ConnectorUtils.class, ctx -> { - ctx.when(() -> ConnectorUtils.getIOSinkClass(any())) - .thenReturn(CASSANDRA_STRING_SINK); - }); - - mockStatic(ClassLoaderUtils.class, ctx -> { - }); - - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.extractFileFromPkgURL(any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getSinkType(any())).thenReturn(String.class); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mock(NarClassLoader.class)); - ctx.when(() -> FunctionCommon - .convertProcessingGuarantee(eq(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE))) - .thenReturn(ATLEAST_ONCE); - }); - this.mockedFunctionMetaData = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetaData); @@ -1219,35 +955,17 @@ public void testUpdateSinkDifferentTransformFunction() throws Exception { SinkConfig sinkConfig = createDefaultSinkConfig(); sinkConfig.setTransformFunction("builtin://transform"); - sinkConfig.setTransformFunctionClassName("DummyFunction"); sinkConfig.setTransformFunctionConfig("{\"dummy\": \"dummy\"}"); - NarClassLoader mockedClassLoader = mock(NarClassLoader.class); - doReturn(RecordFunction.class).when(mockedClassLoader).loadClass("DummyFunction"); - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(FunctionCommon::createPkgTempFile).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getRawFunctionTypes(any(), anyBoolean())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getFunctionTypes(any(), anyBoolean())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getFunctionClassParent(any(), anyBoolean())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mockedClassLoader); - }); + registerBuiltinFunction("transform", RecordFunction.class.getName()); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); this.mockedFunctionMetaData = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))).thenReturn(mockedFunctionMetaData); - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); - FunctionsManager mockedFunctionsManager = mock(FunctionsManager.class); - FunctionArchive functionArchive = FunctionArchive.builder() - .classLoader(mockedClassLoader) - .build(); - when(mockedFunctionsManager.getFunction("transform")).thenReturn(functionArchive); - - when(mockedWorkerService.getFunctionsManager()).thenReturn(mockedFunctionsManager); - - try (FileInputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) { resource.updateSink( tenant, @@ -1737,6 +1455,14 @@ private SinkConfig createDefaultSinkConfig() { return sinkConfig; } + private void mockFunctionCommon(String tenant, String namespace, String sink) throws IOException { + this.mockedFunctionMetaData = + Function.FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); + when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))).thenReturn(mockedFunctionMetaData); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); + } + private FunctionDetails createDefaultFunctionDetails() throws IOException { return SinkConfigUtils.convert(createDefaultSinkConfig(), new SinkConfigUtils.ExtractedSinkDetails(null, null, null)); @@ -1760,21 +1486,7 @@ public void testRegisterSinkSuccessK8sNoUpload() throws Exception { }); - NarClassLoader mockedClassLoader = mock(NarClassLoader.class); - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.getSinkType(any())).thenReturn(String.class); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.isFunctionCodeBuiltin(any())).thenReturn(true); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mockedClassLoader); - }); - - ConnectorsManager mockedConnManager = mock(ConnectorsManager.class); - Connector connector = Connector.builder() - .classLoader(mockedClassLoader) - .build(); - when(mockedConnManager.getConnector("cassandra")).thenReturn(connector); - when(mockedConnManager.getSinkArchive(any())).thenReturn(getPulsarIOCassandraNar().toPath()); - when(mockedWorkerService.getConnectorsManager()).thenReturn(mockedConnManager); + registerBuiltinConnector("cassandra", getPulsarIOCassandraNar()); when(mockedRuntimeFactory.externallyManaged()).thenReturn(true); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(false); @@ -1782,17 +1494,15 @@ public void testRegisterSinkSuccessK8sNoUpload() throws Exception { SinkConfig sinkConfig = createDefaultSinkConfig(); sinkConfig.setArchive("builtin://cassandra"); - try (FileInputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) { - resource.registerSink( - tenant, - namespace, - sink, - inputStream, - mockedFormData, - null, - sinkConfig, - null); - } + resource.registerSink( + tenant, + namespace, + sink, + null, + mockedFormData, + null, + sinkConfig, + null); } /* @@ -1814,23 +1524,7 @@ public void testRegisterSinkSuccessK8sWithUpload() throws Exception { }); - NarClassLoader mockedClassLoader = mock(NarClassLoader.class); - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.getSinkType(any())).thenReturn(String.class); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.isFunctionCodeBuiltin(any())).thenReturn(true); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mockedClassLoader); - }); - - ConnectorsManager mockedConnManager = mock(ConnectorsManager.class); - Connector connector = Connector.builder() - .classLoader(mockedClassLoader) - .build(); - when(mockedConnManager.getConnector("cassandra")).thenReturn(connector); - when(mockedConnManager.getSinkArchive(any())).thenReturn(getPulsarIOCassandraNar().toPath()); - - when(mockedWorkerService.getConnectorsManager()).thenReturn(mockedConnManager); - + registerBuiltinConnector("cassandra", getPulsarIOCassandraNar()); when(mockedRuntimeFactory.externallyManaged()).thenReturn(true); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(false); @@ -1838,21 +1532,19 @@ public void testRegisterSinkSuccessK8sWithUpload() throws Exception { SinkConfig sinkConfig = createDefaultSinkConfig(); sinkConfig.setArchive("builtin://cassandra"); - try (FileInputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) { - try { - resource.registerSink( - tenant, - namespace, - sink, - inputStream, - mockedFormData, - null, - sinkConfig, - null); - Assert.fail(); - } catch (RuntimeException e) { - Assert.assertEquals(e.getMessage(), injectedErrMsg); - } + try { + resource.registerSink( + tenant, + namespace, + sink, + null, + mockedFormData, + null, + sinkConfig, + null); + Assert.fail(); + } catch (RuntimeException e) { + Assert.assertEquals(e.getMessage(), injectedErrMsg); } } @@ -1865,11 +1557,6 @@ public void testUpdateSinkWithNoChange() throws IOException { mockStatic(SinkConfigUtils.class, ctx -> { ctx.when(() -> SinkConfigUtils.convertFromDetails(any())).thenReturn(sinkConfig); - ctx.when(() -> SinkConfigUtils.convert(any(), any())).thenCallRealMethod(); - ctx.when(() -> SinkConfigUtils.validateUpdate(any(), any())).thenCallRealMethod(); - ctx.when(() -> SinkConfigUtils.clone(any())).thenCallRealMethod(); - ctx.when(() -> SinkConfigUtils.collectAllInputTopics(any())).thenCallRealMethod(); - ctx.when(() -> SinkConfigUtils.validateAndExtractDetails(any(),any(),any(),anyBoolean())).thenCallRealMethod(); }); mockFunctionCommon(sinkConfig.getTenant(), sinkConfig.getNamespace(), sinkConfig.getName()); @@ -1912,15 +1599,17 @@ public void testUpdateSinkWithNoChange() throws IOException { // no changes but set the auth-update flag to true, should not fail UpdateOptionsImpl updateOptions = new UpdateOptionsImpl(); updateOptions.setUpdateAuthData(true); - resource.updateSink( - sinkConfig.getTenant(), - sinkConfig.getNamespace(), - sinkConfig.getName(), - null, - mockedFormData, - null, - sinkConfig, - null, - updateOptions); + try (FileInputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) { + resource.updateSink( + sinkConfig.getTenant(), + sinkConfig.getNamespace(), + sinkConfig.getName(), + inputStream, + mockedFormData, + null, + sinkConfig, + null, + updateOptions); + } } } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java index eabf954cc77b1..f02acbd3663bf 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java @@ -18,17 +18,10 @@ */ package org.apache.pulsar.functions.worker.rest.api.v3; -import static org.apache.pulsar.functions.worker.rest.api.v3.SinkApiV3ResourceTest.getPulsarIOCassandraNar; -import static org.apache.pulsar.functions.worker.rest.api.v3.SinkApiV3ResourceTest.getPulsarIOInvalidNar; -import static org.apache.pulsar.functions.worker.rest.api.v3.SinkApiV3ResourceTest.getPulsarIOTwitterNar; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.when; @@ -41,31 +34,18 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.util.HashMap; +import java.lang.reflect.Method; import java.util.LinkedList; import java.util.List; -import java.util.Map; -import java.util.function.Consumer; import javax.ws.rs.core.Response; import org.apache.distributedlog.api.namespace.Namespace; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; import org.apache.pulsar.broker.authentication.AuthenticationParameters; -import org.apache.pulsar.client.admin.Functions; -import org.apache.pulsar.client.admin.Namespaces; -import org.apache.pulsar.client.admin.Packages; -import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.admin.Tenants; import org.apache.pulsar.common.functions.UpdateOptionsImpl; import org.apache.pulsar.common.functions.Utils; import org.apache.pulsar.common.io.SourceConfig; -import org.apache.pulsar.common.nar.NarClassLoader; -import org.apache.pulsar.common.nar.NarClassLoaderBuilder; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.util.ClassLoaderUtils; import org.apache.pulsar.common.util.RestException; import org.apache.pulsar.functions.api.utils.IdentityFunction; @@ -75,159 +55,43 @@ import org.apache.pulsar.functions.proto.Function.ProcessingGuarantees; import org.apache.pulsar.functions.proto.Function.SinkSpec; import org.apache.pulsar.functions.proto.Function.SourceSpec; -import org.apache.pulsar.functions.runtime.RuntimeFactory; import org.apache.pulsar.functions.source.TopicSchema; -import org.apache.pulsar.functions.utils.FunctionCommon; import org.apache.pulsar.functions.utils.SourceConfigUtils; import org.apache.pulsar.functions.utils.io.ConnectorUtils; -import org.apache.pulsar.functions.worker.FunctionMetaDataManager; -import org.apache.pulsar.functions.worker.FunctionRuntimeManager; -import org.apache.pulsar.functions.worker.LeaderService; -import org.apache.pulsar.functions.worker.PulsarWorkerService; import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerUtils; -import org.apache.pulsar.functions.worker.rest.api.PulsarFunctionTestTemporaryDirectory; import org.apache.pulsar.functions.worker.rest.api.SourcesImpl; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.mockito.MockedStatic; import org.mockito.Mockito; -import org.testng.annotations.AfterClass; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; /** * Unit test of {@link SourcesApiV3Resource}. */ -public class SourceApiV3ResourceTest { +public class SourceApiV3ResourceTest extends AbstractFunctionsResourceTest { - private static final String tenant = "test-tenant"; - private static final String namespace = "test-namespace"; private static final String source = "test-source"; private static final String outputTopic = "test-output-topic"; private static final String outputSerdeClassName = TopicSchema.DEFAULT_SERDE; private static final String TWITTER_FIRE_HOSE = "org.apache.pulsar.io.twitter.TwitterFireHose"; - private static final int parallelism = 1; - - private PulsarWorkerService mockedWorkerService; - private PulsarAdmin mockedPulsarAdmin; - private Tenants mockedTenants; - private Namespaces mockedNamespaces; - private Functions mockedFunctions; - private TenantInfoImpl mockedTenantInfo; - private List namespaceList = new LinkedList<>(); - private FunctionMetaDataManager mockedManager; - private FunctionRuntimeManager mockedFunctionRunTimeManager; - private RuntimeFactory mockedRuntimeFactory; - private Namespace mockedNamespace; private SourcesImpl resource; - private InputStream mockedInputStream; - private FormDataContentDisposition mockedFormData; - private FunctionMetaData mockedFunctionMetaData; - private LeaderService mockedLeaderService; - private Packages mockedPackages; - private PulsarFunctionTestTemporaryDirectory tempDirectory; - - private static NarClassLoader narClassLoader; - private static Map mockStaticContexts = new HashMap<>(); - - @BeforeClass - public void setupNarClassLoader() throws IOException { - narClassLoader = NarClassLoaderBuilder.builder().narFile(getPulsarIOTwitterNar()).build(); - } - - @AfterClass(alwaysRun = true) - public void cleanupNarClassLoader() throws IOException { - if (narClassLoader != null) { - narClassLoader.close(); - narClassLoader = null; - } - } - - @BeforeMethod - public void setup() throws Exception { - this.mockedManager = mock(FunctionMetaDataManager.class); - this.mockedFunctionRunTimeManager = mock(FunctionRuntimeManager.class); - this.mockedRuntimeFactory = mock(RuntimeFactory.class); - this.mockedInputStream = mock(InputStream.class); - this.mockedNamespace = mock(Namespace.class); - this.mockedFormData = mock(FormDataContentDisposition.class); - when(mockedFormData.getFileName()).thenReturn("test"); - this.mockedTenantInfo = mock(TenantInfoImpl.class); - this.mockedPulsarAdmin = mock(PulsarAdmin.class); - this.mockedTenants = mock(Tenants.class); - this.mockedNamespaces = mock(Namespaces.class); - this.mockedFunctions = mock(Functions.class); - this.mockedLeaderService = mock(LeaderService.class); - this.mockedPackages = mock(Packages.class); - namespaceList.add(tenant + "/" + namespace); - - this.mockedWorkerService = mock(PulsarWorkerService.class); - when(mockedWorkerService.getFunctionMetaDataManager()).thenReturn(mockedManager); - when(mockedWorkerService.getLeaderService()).thenReturn(mockedLeaderService); - when(mockedWorkerService.getFunctionRuntimeManager()).thenReturn(mockedFunctionRunTimeManager); - when(mockedFunctionRunTimeManager.getRuntimeFactory()).thenReturn(mockedRuntimeFactory); - when(mockedWorkerService.getDlogNamespace()).thenReturn(mockedNamespace); - when(mockedWorkerService.isInitialized()).thenReturn(true); - when(mockedWorkerService.getBrokerAdmin()).thenReturn(mockedPulsarAdmin); - when(mockedWorkerService.getFunctionAdmin()).thenReturn(mockedPulsarAdmin); - when(mockedPulsarAdmin.tenants()).thenReturn(mockedTenants); - when(mockedPulsarAdmin.namespaces()).thenReturn(mockedNamespaces); - when(mockedPulsarAdmin.functions()).thenReturn(mockedFunctions); - when(mockedPulsarAdmin.packages()).thenReturn(mockedPackages); - when(mockedTenants.getTenantInfo(any())).thenReturn(mockedTenantInfo); - when(mockedNamespaces.getNamespaces(any())).thenReturn(namespaceList); - when(mockedLeaderService.isLeader()).thenReturn(true); - doAnswer(invocationOnMock -> { - Files.copy(getPulsarIOTwitterNar().toPath(), Paths.get(invocationOnMock.getArgument(1, String.class)), - StandardCopyOption.REPLACE_EXISTING); - return null; - }).when(mockedPackages).download(any(), any()); - - // worker config - WorkerConfig workerConfig = new WorkerConfig() - .setWorkerId("test") - .setWorkerPort(8080) - .setFunctionMetadataTopicName("pulsar/functions") - .setNumFunctionPackageReplicas(3) - .setPulsarServiceUrl("pulsar://localhost:6650/"); - tempDirectory = PulsarFunctionTestTemporaryDirectory.create(getClass().getSimpleName()); - tempDirectory.useTemporaryDirectoriesForWorkerConfig(workerConfig); - when(mockedWorkerService.getWorkerConfig()).thenReturn(workerConfig); + @Override + protected void doSetup() { this.resource = spy(new SourcesImpl(() -> mockedWorkerService)); } - private void mockStatic(Class classStatic, Consumer> consumer) { - final MockedStatic mockedStatic = - mockStaticContexts.computeIfAbsent(classStatic.getName(), name -> Mockito.mockStatic(classStatic)); - consumer.accept(mockedStatic); - } - - @AfterMethod(alwaysRun = true) - public void cleanup() { - if (tempDirectory != null) { - tempDirectory.delete(); + @Override + protected void customizeWorkerConfig(WorkerConfig workerConfig, Method method) { + if (method.getName().endsWith("UploadFailure") || method.getName().contains("BKPackage")) { + workerConfig.setFunctionsWorkerEnablePackageManagement(false); } - mockStaticContexts.values().forEach(MockedStatic::close); - mockStaticContexts.clear(); } - private void mockWorkerUtils() { - mockStatic(WorkerUtils.class, - ctx -> { - ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); - }); - } - - private void mockWorkerUtils(Consumer> consumer) { - mockStatic(WorkerUtils.class, ctx -> { - ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); - if (consumer != null) { - consumer.accept(ctx); - } - }); + @Override + protected FunctionDetails.ComponentType getComponentType() { + return FunctionDetails.ComponentType.SOURCE; } // @@ -297,8 +161,8 @@ public void testRegisterSourceMissingSourceName() { } } - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Source class UnknownClass must" - + " be in class path") + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Source class UnknownClass not " + + "found in class loader") public void testRegisterSourceWrongClassName() { try { testRegisterSourceMissingArguments( @@ -361,10 +225,8 @@ public void testRegisterSourceMissingPackageDetails() throws IOException { } } - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Source package does not have the" - + " correct format. Pulsar cannot determine if the package is a NAR package" - + " or JAR package. Source classname is not provided and attempts to load it as a NAR package " - + "produced the following error.") + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Source package doesn't contain" + + " the META-INF/services/pulsar-io.yaml file.") public void testRegisterSourceMissingPackageDetailsAndClassname() { try { testRegisterSourceMissingArguments( @@ -385,8 +247,8 @@ public void testRegisterSourceMissingPackageDetailsAndClassname() { } } - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Failed to extract source class" - + " from archive") + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Source package doesn't contain" + + " the META-INF/services/pulsar-io.yaml file.") public void testRegisterSourceInvalidJarWithNoSource() throws IOException { try (InputStream inputStream = new FileInputStream(getPulsarIOInvalidNar())) { testRegisterSourceMissingArguments( @@ -524,7 +386,7 @@ public void testUpdateMissingSinkConfig() { } private void registerDefaultSource() throws IOException { - registerDefaultSourceWithPackageUrl("source://public/default/test@v1"); + registerDefaultSourceWithPackageUrl(getPulsarIOTwitterNar().toURI().toString()); } private void registerDefaultSourceWithPackageUrl(String packageUrl) throws IOException { @@ -565,8 +427,6 @@ public void testRegisterSourceUploadFailure() throws Exception { any(File.class), any(Namespace.class))) .thenThrow(new IOException("upload failure")); - - ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); }); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(false); @@ -593,7 +453,7 @@ public void testRegisterSourceSuccess() throws Exception { @Test(timeOut = 20000) public void testRegisterSourceSuccessWithPackageName() throws IOException { - registerDefaultSourceWithPackageUrl("source://public/default/test@v1"); + registerDefaultSource(); } @Test(timeOut = 20000) @@ -621,14 +481,7 @@ public void testRegisterSourceConflictingFields() throws Exception { when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(true); when(mockedManager.containsFunction(eq(actualTenant), eq(actualNamespace), eq(actualName))).thenReturn(false); - SourceConfig sourceConfig = new SourceConfig(); - sourceConfig.setTenant(tenant); - sourceConfig.setNamespace(namespace); - sourceConfig.setName(source); - sourceConfig.setClassName(TWITTER_FIRE_HOSE); - sourceConfig.setParallelism(parallelism); - sourceConfig.setTopicName(outputTopic); - sourceConfig.setSerdeClassName(outputSerdeClassName); + SourceConfig sourceConfig = createDefaultSourceConfig(); try (InputStream inputStream = new FileInputStream(getPulsarIOTwitterNar())) { resource.registerSource( actualTenant, @@ -815,17 +668,19 @@ public void testUpdateSourceChangedParallelism() throws Exception { try { mockWorkerUtils(); - testUpdateSourceMissingArguments( - tenant, - namespace, - source, - null, - mockedFormData, - outputTopic, - outputSerdeClassName, - TWITTER_FIRE_HOSE, - parallelism + 1, - null); + try(FileInputStream inputStream = new FileInputStream(getPulsarIOTwitterNar())) { + testUpdateSourceMissingArguments( + tenant, + namespace, + source, + inputStream, + mockedFormData, + outputTopic, + outputSerdeClassName, + TWITTER_FIRE_HOSE, + parallelism + 1, + null); + } } catch (RestException re) { assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); throw re; @@ -836,31 +691,29 @@ public void testUpdateSourceChangedParallelism() throws Exception { public void testUpdateSourceChangedTopic() throws Exception { mockWorkerUtils(); - testUpdateSourceMissingArguments( - tenant, - namespace, - source, - null, - mockedFormData, - "DifferentTopic", - outputSerdeClassName, - TWITTER_FIRE_HOSE, - parallelism, - null); + try(FileInputStream inputStream = new FileInputStream(getPulsarIOTwitterNar())) { + testUpdateSourceMissingArguments( + tenant, + namespace, + source, + inputStream, + mockedFormData, + "DifferentTopic", + outputSerdeClassName, + TWITTER_FIRE_HOSE, + parallelism, + null); + } } @Test - public void testUpdateSourceWithNoChange() { + public void testUpdateSourceWithNoChange() throws IOException { mockWorkerUtils(); // No change on config, SourceConfig sourceConfig = createDefaultSourceConfig(); mockStatic(SourceConfigUtils.class, ctx -> { ctx.when(() -> SourceConfigUtils.convertFromDetails(any())).thenReturn(sourceConfig); - ctx.when(() -> SourceConfigUtils.convert(any(), any())).thenCallRealMethod(); - ctx.when(() -> SourceConfigUtils.validateUpdate(any(), any())).thenCallRealMethod(); - ctx.when(() -> SourceConfigUtils.clone(any())).thenCallRealMethod(); - ctx.when(() -> SourceConfigUtils.validateAndExtractDetails(any(),any(),anyBoolean())).thenCallRealMethod(); }); mockFunctionCommon(sourceConfig.getTenant(), sourceConfig.getNamespace(), sourceConfig.getName()); @@ -903,16 +756,18 @@ public void testUpdateSourceWithNoChange() { // no changes but set the auth-update flag to true, should not fail UpdateOptionsImpl updateOptions = new UpdateOptionsImpl(); updateOptions.setUpdateAuthData(true); - resource.updateSource( - sourceConfig.getTenant(), - sourceConfig.getNamespace(), - sourceConfig.getName(), - null, - mockedFormData, - null, - sourceConfig, - null, - updateOptions); + try (InputStream inputStream = new FileInputStream(getPulsarIOTwitterNar())) { + resource.updateSource( + sourceConfig.getTenant(), + sourceConfig.getNamespace(), + sourceConfig.getName(), + inputStream, + mockedFormData, + null, + sourceConfig, + null, + updateOptions); + } } @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Source parallelism must be a " @@ -997,14 +852,6 @@ private void mockFunctionCommon(String tenant, String namespace, String function }); mockStatic(ClassLoaderUtils.class, c -> { }); - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.createPkgTempFile()).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getSourceType(argThat(clazz -> clazz.getName().equals(TWITTER_FIRE_HOSE)))) - .thenReturn(String.class); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())) - .thenReturn(narClassLoader); - }); this.mockedFunctionMetaData = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); @@ -1014,49 +861,25 @@ private void mockFunctionCommon(String tenant, String namespace, String function } private void updateDefaultSource() throws Exception { - updateDefaultSourceWithPackageUrl(null); + updateDefaultSourceWithPackageUrl(getPulsarIOTwitterNar().toURI().toString()); } private void updateDefaultSourceWithPackageUrl(String packageUrl) throws Exception { - SourceConfig sourceConfig = new SourceConfig(); - sourceConfig.setTenant(tenant); - sourceConfig.setNamespace(namespace); - sourceConfig.setName(source); - sourceConfig.setClassName(TWITTER_FIRE_HOSE); - sourceConfig.setParallelism(parallelism); - sourceConfig.setTopicName(outputTopic); - sourceConfig.setSerdeClassName(outputSerdeClassName); - - mockStatic(ConnectorUtils.class, c -> { - }); - - mockStatic(ClassLoaderUtils.class, c -> { - }); - - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.createPkgTempFile()).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getSourceType(argThat(clazz -> clazz.getName().equals(TWITTER_FIRE_HOSE)))) - .thenReturn(String.class); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())) - .thenReturn(narClassLoader); - }); + SourceConfig sourceConfig = createDefaultSourceConfig(); this.mockedFunctionMetaData = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetaData); - try (InputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) { - resource.updateSource( - tenant, - namespace, - source, - inputStream, - mockedFormData, - packageUrl, - sourceConfig, - null, null); - } + resource.updateSource( + tenant, + namespace, + source, + null, + mockedFormData, + packageUrl, + sourceConfig, + null, null); } @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Source test-source doesn't " + @@ -1079,11 +902,25 @@ public void testUpdateSourceUploadFailure() throws Exception { anyString(), any(File.class), any(Namespace.class))).thenThrow(new IOException("upload failure")); - ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); }); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(true); - updateDefaultSource(); + SourceConfig sourceConfig = createDefaultSourceConfig(); + this.mockedFunctionMetaData = + FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); + when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetaData); + + try(InputStream inputStream = new FileInputStream(getPulsarIOTwitterNar())) { + resource.updateSource( + tenant, + namespace, + source, + inputStream, + mockedFormData, + null, + sourceConfig, + null, null); + } } catch (RestException re) { assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); throw re; @@ -1103,16 +940,9 @@ public void testUpdateSourceSuccess() throws Exception { public void testUpdateSourceWithUrl() throws Exception { Configurator.setRootLevel(Level.DEBUG); - String filePackageUrl = getPulsarIOCassandraNar().toURI().toString(); + String filePackageUrl = getPulsarIOTwitterNar().toURI().toString(); - SourceConfig sourceConfig = new SourceConfig(); - sourceConfig.setTopicName(outputTopic); - sourceConfig.setSerdeClassName(outputSerdeClassName); - sourceConfig.setTenant(tenant); - sourceConfig.setNamespace(namespace); - sourceConfig.setName(source); - sourceConfig.setClassName(TWITTER_FIRE_HOSE); - sourceConfig.setParallelism(parallelism); + SourceConfig sourceConfig = createDefaultSourceConfig(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(true); mockStatic(ConnectorUtils.class, c -> { @@ -1120,15 +950,6 @@ public void testUpdateSourceWithUrl() throws Exception { mockStatic(ClassLoaderUtils.class, c -> { }); - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.extractFileFromPkgURL(any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getSourceType(argThat(clazz -> clazz.getName().equals(TWITTER_FIRE_HOSE)))) - .thenReturn(String.class); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())) - .thenReturn(narClassLoader); - }); - this.mockedFunctionMetaData = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetaData); diff --git a/pulsar-io/aerospike/pom.xml b/pulsar-io/aerospike/pom.xml index 9bed409ce3742..b89ff2bf8cd71 100644 --- a/pulsar-io/aerospike/pom.xml +++ b/pulsar-io/aerospike/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.2.0-SNAPSHOT - pulsar-io-aerospike Pulsar IO :: Aerospike - - ${project.groupId} pulsar-io-core ${project.version} - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.aerospike aerospike-client-bc @@ -63,9 +57,7 @@ org.bouncycastle bcpkix-jdk18on - - @@ -91,6 +83,4 @@ - - diff --git a/pulsar-io/aerospike/src/main/resources/findbugsExclude.xml b/pulsar-io/aerospike/src/main/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-io/aerospike/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/aerospike/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-io/alluxio/pom.xml b/pulsar-io/alluxio/pom.xml index 8abf89a98aab2..275ee30c6eca2 100644 --- a/pulsar-io/alluxio/pom.xml +++ b/pulsar-io/alluxio/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - pulsar-io - org.apache.pulsar - 3.2.0-SNAPSHOT - - - - 2.9.3 - 4.1.11 - 1.37.0 - 4.1.100.Final - - - pulsar-io-alluxio - Pulsar IO :: Alluxio - + + 4.0.0 + + pulsar-io + io.streamnative + 3.2.0-SNAPSHOT + + + 2.9.3 + 4.1.11 + 1.37.0 + 4.1.100.Final + + pulsar-io-alluxio + Pulsar IO :: Alluxio + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + ${project.groupId} + pulsar-client-original + ${project.version} + test + + + org.alluxio + alluxio-core-client-fs + ${alluxio.version} + + + org.alluxio + alluxio-minicluster + ${alluxio.version} + test + + + org.glassfish + javax.el + + + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + com.google.guava + guava + + + - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - - ${project.groupId} - pulsar-client-original - ${project.version} - test - - - - org.alluxio - alluxio-core-client-fs - ${alluxio.version} - - - - org.alluxio - alluxio-minicluster - ${alluxio.version} - test - - - org.glassfish - javax.el - - - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - com.google.guava - guava - + + io.netty + netty-bom + ${netty.version} + pom + import + + + io.grpc + grpc-bom + ${grpc.version} + pom + import + + + io.dropwizard.metrics + metrics-jvm + ${metrics.version} + - - - - - io.netty - netty-bom - ${netty.version} - pom - import - - - io.grpc - grpc-bom - ${grpc.version} - pom - import - - - io.dropwizard.metrics - metrics-jvm - ${metrics.version} - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + diff --git a/pulsar-io/aws/pom.xml b/pulsar-io/aws/pom.xml index 18bdd6de8746a..f49a712d79084 100644 --- a/pulsar-io/aws/pom.xml +++ b/pulsar-io/aws/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.2.0-SNAPSHOT - pulsar-io-aws Pulsar IO :: IO AWS - ${project.groupId} pulsar-io-core ${project.version} - com.google.code.gson gson - com.amazonaws aws-java-sdk-sts - software.amazon.awssdk sts 2.10.56 - - + - diff --git a/pulsar-io/aws/src/main/resources/findbugsExclude.xml b/pulsar-io/aws/src/main/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-io/aws/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/aws/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-io/batch-data-generator/pom.xml b/pulsar-io/batch-data-generator/pom.xml index 3b76e36a1b21d..905c0dce496e3 100644 --- a/pulsar-io/batch-data-generator/pom.xml +++ b/pulsar-io/batch-data-generator/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io - 3.2.0-SNAPSHOT - - - pulsar-io-batch-data-generator - Pulsar IO :: Batch Data Generator - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - - ${project.groupId} - pulsar-io-batch-discovery-triggerers - ${project.version} - - - - org.springframework - spring-context - ${spring.version} - - - - io.codearte.jfairy - jfairy - 0.5.9 - - - - org.apache.avro - avro - ${avro.version} - - - - ${project.groupId} - pulsar-functions-local-runner-original - ${project.version} - test - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - spotbugs - verify - - check - - - - - - + + 4.0.0 + + io.streamnative + pulsar-io + 3.2.0-SNAPSHOT + + pulsar-io-batch-data-generator + Pulsar IO :: Batch Data Generator + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + ${project.groupId} + pulsar-io-batch-discovery-triggerers + ${project.version} + + + org.springframework + spring-context + ${spring.version} + + + io.codearte.jfairy + jfairy + 0.5.9 + + + org.apache.avro + avro + ${avro.version} + + + ${project.groupId} + pulsar-functions-local-runner-original + ${project.version} + test + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + spotbugs + verify + + check + + + + + + diff --git a/pulsar-io/batch-data-generator/src/main/resources/findbugsExclude.xml b/pulsar-io/batch-data-generator/src/main/resources/findbugsExclude.xml index 0b1652ef62881..1d0d64fba7e67 100644 --- a/pulsar-io/batch-data-generator/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/batch-data-generator/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-io 3.2.0-SNAPSHOT - pulsar-io-batch-discovery-triggerers Pulsar IO :: Batch Discovery Triggerers - ${project.groupId} pulsar-io-core ${project.version} - com.cronutils cron-utils ${cron-utils.version} - org.springframework spring-context ${spring.version} - - @@ -73,5 +66,4 @@ - diff --git a/pulsar-io/batch-discovery-triggerers/src/main/resources/findbugsExclude.xml b/pulsar-io/batch-discovery-triggerers/src/main/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-io/batch-discovery-triggerers/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/batch-discovery-triggerers/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - org.apache.pulsar - pulsar-io - 3.2.0-SNAPSHOT - - 4.0.0 - - pulsar-io-canal - Pulsar IO :: Canal - - - 1.1.5 - - - - - ${project.groupId} - pulsar-io-common - ${project.version} - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - com.alibaba - fastjson - 1.2.83 - - - - org.springframework - spring-core - ${spring.version} - - - org.springframework - spring-aop - ${spring.version} - - - org.springframework - spring-context - ${spring.version} - - - org.springframework - spring-jdbc - ${spring.version} - - - org.springframework - spring-orm - ${spring.version} - - - - com.alibaba.otter - canal.protocol - ${canal.version} - - - ch.qos.logback - * - - - - - com.alibaba.otter - canal.client - ${canal.version} - - - ch.qos.logback - * - - - org.springframework - * - - - - - - org.apache.logging.log4j - log4j-core - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - - - \ No newline at end of file + + + io.streamnative + pulsar-io + 3.2.0-SNAPSHOT + + 4.0.0 + pulsar-io-canal + Pulsar IO :: Canal + + 1.1.5 + + + + ${project.groupId} + pulsar-io-common + ${project.version} + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + com.alibaba + fastjson + 1.2.83 + + + org.springframework + spring-core + ${spring.version} + + + org.springframework + spring-aop + ${spring.version} + + + org.springframework + spring-context + ${spring.version} + + + org.springframework + spring-jdbc + ${spring.version} + + + org.springframework + spring-orm + ${spring.version} + + + com.alibaba.otter + canal.protocol + ${canal.version} + + + ch.qos.logback + * + + + + + com.alibaba.otter + canal.client + ${canal.version} + + + ch.qos.logback + * + + + org.springframework + * + + + + + org.apache.logging.log4j + log4j-core + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + + diff --git a/pulsar-io/cassandra/pom.xml b/pulsar-io/cassandra/pom.xml index efe030307db8f..1e2d0f92d5be1 100644 --- a/pulsar-io/cassandra/pom.xml +++ b/pulsar-io/cassandra/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.2.0-SNAPSHOT - pulsar-io-cassandra Pulsar IO :: Cassandra - - ${project.groupId} pulsar-io-core ${project.version} - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.datastax.cassandra cassandra-driver-core - - diff --git a/pulsar-io/common/pom.xml b/pulsar-io/common/pom.xml index db82c3c11a871..72a97563cb520 100644 --- a/pulsar-io/common/pom.xml +++ b/pulsar-io/common/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar - pulsar-io - 3.2.0-SNAPSHOT - - - pulsar-io-common - Pulsar IO :: IO Common - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - com.fasterxml.jackson.core - jackson-databind - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - spotbugs - verify - - check - - - - - - - + + 4.0.0 + + io.streamnative + pulsar-io + 3.2.0-SNAPSHOT + + pulsar-io-common + Pulsar IO :: IO Common + + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + spotbugs + verify + + check + + + + + + diff --git a/pulsar-io/common/src/main/resources/findbugsExclude.xml b/pulsar-io/common/src/main/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-io/common/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/common/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.2.0-SNAPSHOT - pulsar-io-core Pulsar IO :: IO - ${project.groupId} @@ -37,7 +35,6 @@ ${project.version} - @@ -59,5 +56,4 @@ - diff --git a/pulsar-io/core/src/main/resources/findbugsExclude.xml b/pulsar-io/core/src/main/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-io/core/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/core/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io - 3.2.0-SNAPSHOT - - - pulsar-io-data-generator - Pulsar IO :: Data Generator - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - - ${project.groupId} - pulsar-config-validation - ${project.version} - - - - io.codearte.jfairy - jfairy - 0.5.9 - - - - org.apache.avro - avro - ${avro.version} - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - + + 4.0.0 + + io.streamnative + pulsar-io + 3.2.0-SNAPSHOT + + pulsar-io-data-generator + Pulsar IO :: Data Generator + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + ${project.groupId} + pulsar-config-validation + ${project.version} + + + io.codearte.jfairy + jfairy + 0.5.9 + + + org.apache.avro + avro + ${avro.version} + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + diff --git a/pulsar-io/debezium/core/pom.xml b/pulsar-io/debezium/core/pom.xml index f0ec085424276..bed2eed1851c5 100644 --- a/pulsar-io/debezium/core/pom.xml +++ b/pulsar-io/debezium/core/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io-debezium 3.2.0-SNAPSHOT - pulsar-io-debezium-core Pulsar IO :: Debezium :: Core - - ${project.groupId} pulsar-io-core ${project.version} provided - io.debezium debezium-core ${debezium.version} - org.apache.commons commons-lang3 - ${project.groupId} pulsar-common ${project.version} - ${project.groupId} pulsar-io-kafka-connect-adaptor ${project.version} - org.apache.kafka connect-runtime @@ -77,21 +69,18 @@ - ${project.groupId} pulsar-broker ${project.version} test - ${project.groupId} testmocks ${project.version} test - ${project.groupId} pulsar-broker @@ -99,21 +88,17 @@ test test-jar - io.debezium debezium-connector-mysql ${debezium.version} test - ${project.groupId} pulsar-client-original ${project.version} test - - diff --git a/pulsar-io/debezium/mongodb/pom.xml b/pulsar-io/debezium/mongodb/pom.xml index 4608261b90953..b30c2d025f76c 100644 --- a/pulsar-io/debezium/mongodb/pom.xml +++ b/pulsar-io/debezium/mongodb/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io-debezium - 3.2.0-SNAPSHOT - - - pulsar-io-debezium-mongodb - Pulsar IO :: Debezium :: mongodb - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - provided - - - - ${project.groupId} - pulsar-io-debezium-core - ${project.version} - - - - io.debezium - debezium-connector-mongodb - ${debezium.version} - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - + + 4.0.0 + + io.streamnative + pulsar-io-debezium + 3.2.0-SNAPSHOT + + pulsar-io-debezium-mongodb + Pulsar IO :: Debezium :: mongodb + + + ${project.groupId} + pulsar-io-core + ${project.version} + provided + + + ${project.groupId} + pulsar-io-debezium-core + ${project.version} + + + io.debezium + debezium-connector-mongodb + ${debezium.version} + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + diff --git a/pulsar-io/debezium/mssql/pom.xml b/pulsar-io/debezium/mssql/pom.xml index 16c89149bbc0e..c1e9fed2defab 100644 --- a/pulsar-io/debezium/mssql/pom.xml +++ b/pulsar-io/debezium/mssql/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io-debezium 3.2.0-SNAPSHOT - pulsar-io-debezium-mssql Pulsar IO :: Debezium :: Microsoft SQL - - ${project.groupId} pulsar-io-core ${project.version} provided - ${project.groupId} pulsar-io-debezium-core ${project.version} - io.debezium debezium-connector-sqlserver ${debezium.version} - - @@ -61,5 +54,4 @@ - diff --git a/pulsar-io/debezium/mysql/pom.xml b/pulsar-io/debezium/mysql/pom.xml index 1cc9777fee235..ff9c931a47610 100644 --- a/pulsar-io/debezium/mysql/pom.xml +++ b/pulsar-io/debezium/mysql/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io-debezium 3.2.0-SNAPSHOT - pulsar-io-debezium-mysql Pulsar IO :: Debezium :: mysql - ${project.groupId} @@ -37,21 +35,17 @@ ${project.version} provided - ${project.groupId} pulsar-io-debezium-core ${project.version} - io.debezium debezium-connector-mysql ${debezium.version} - - @@ -61,8 +55,6 @@ - - @@ -71,5 +63,4 @@ - diff --git a/pulsar-io/debezium/oracle/pom.xml b/pulsar-io/debezium/oracle/pom.xml index 662b7f677183b..69d3104897689 100644 --- a/pulsar-io/debezium/oracle/pom.xml +++ b/pulsar-io/debezium/oracle/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io-debezium 3.2.0-SNAPSHOT - pulsar-io-debezium-oracle Pulsar IO :: Debezium :: oracle - - ${project.groupId} pulsar-io-core ${project.version} provided - ${project.groupId} pulsar-io-debezium-core ${project.version} - io.debezium debezium-connector-oracle ${debezium.version} - - @@ -61,5 +54,4 @@ - diff --git a/pulsar-io/debezium/pom.xml b/pulsar-io/debezium/pom.xml index 04de3c460eef8..9ef472f8c25da 100644 --- a/pulsar-io/debezium/pom.xml +++ b/pulsar-io/debezium/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar-io 3.2.0-SNAPSHOT - pulsar-io-debezium Pulsar IO :: Debezium - @@ -70,7 +68,6 @@ - core mysql @@ -79,5 +76,4 @@ oracle mssql - diff --git a/pulsar-io/debezium/postgres/pom.xml b/pulsar-io/debezium/postgres/pom.xml index 2d7fe02154b48..417b412dc201e 100644 --- a/pulsar-io/debezium/postgres/pom.xml +++ b/pulsar-io/debezium/postgres/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io-debezium 3.2.0-SNAPSHOT - pulsar-io-debezium-postgres Pulsar IO :: Debezium :: postgres - - ${project.groupId} pulsar-io-core ${project.version} provided - ${project.groupId} pulsar-io-debezium-core ${project.version} - io.debezium debezium-connector-postgres ${debezium.version} - org.postgresql postgresql ${debezium.postgresql.version} runtime - - @@ -68,5 +60,4 @@ - diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index fa72146efdd2f..0a3fdb27a6015 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.2.0-SNAPSHOT - pulsar-io-docs Pulsar IO :: Docs - - ${project.groupId} pulsar-io-core @@ -45,7 +42,6 @@ com.beust jcommander - ${project.groupId} @@ -218,7 +214,6 @@ ${project.version} - @@ -267,5 +262,4 @@ - diff --git a/pulsar-io/docs/src/main/resources/findbugsExclude.xml b/pulsar-io/docs/src/main/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-io/docs/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/docs/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.2.0-SNAPSHOT - pulsar-io-dynamodb Pulsar IO :: DynamoDB - - ${project.groupId} pulsar-io-common ${project.version} - ${project.groupId} pulsar-io-core ${project.version} - ${project.groupId} pulsar-functions-instance ${project.version} provided - ${project.groupId} pulsar-io-aws ${project.version} - org.apache.commons commons-lang3 - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.fasterxml.jackson.dataformat jackson-dataformat-cbor - com.google.code.gson gson - com.amazonaws aws-java-sdk-core - com.amazonaws @@ -96,9 +83,7 @@ 1.5.1 - - @@ -107,5 +92,4 @@ - diff --git a/pulsar-io/elastic-search/pom.xml b/pulsar-io/elastic-search/pom.xml index e57c345cbc450..d3bb92c6edd97 100644 --- a/pulsar-io/elastic-search/pom.xml +++ b/pulsar-io/elastic-search/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io - 3.2.0-SNAPSHOT - - - pulsar-io-flume - Pulsar IO :: Flume - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - org.apache.flume - flume-ng-node - 1.11.0 - pom - - - org.apache.avro - avro-ipc - - - avro - org.apache.avro - - - - - org.apache.avro - avro - ${avro.version} - - - commons-lang - commons-lang - 2.6 - - - org.apache.avro - avro-ipc - ${avro.version} - - - org.mortbay.jetty - servlet-api - - - io.netty - netty - - - - - org.apache.curator - curator-framework - ${curator.version} - - - org.apache.curator - curator-test - ${curator.version} - test - - - - io.dropwizard.metrics - metrics-core - test - - - - org.xerial.snappy - snappy-java - test - - - - com.google.guava - guava - ${guava.version} - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - - + + io.dropwizard.metrics + metrics-core + test + + + + org.xerial.snappy + snappy-java + test + + + com.google.guava + guava + ${guava.version} + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + + + - - owasp-dependency-check - - - - org.owasp - dependency-check-maven - ${dependency-check-maven.version} - - - - aggregate - - none - - - - - - - - + + owasp-dependency-check + + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + + + aggregate + + none + + + + + + + diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml index bfce4050ad3a7..e94830d267ce4 100644 --- a/pulsar-io/hbase/pom.xml +++ b/pulsar-io/hbase/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - pulsar-io - org.apache.pulsar - 3.2.0-SNAPSHOT - - pulsar-io-hbase - Pulsar IO :: Hbase - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - - ${project.groupId} - pulsar-functions-instance - ${project.version} - - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - com.google.guava - guava - - - - org.apache.hbase - hbase-client - ${hbase.version} - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - - - org.apache.hbase - hbase-common - ${hbase.version} - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - - - - owasp-dependency-check - - - - org.owasp - dependency-check-maven - ${dependency-check-maven.version} - - - - aggregate - - none - - - - - - - - + + owasp-dependency-check + + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + + + aggregate + + none + + + + + + + diff --git a/pulsar-io/hbase/src/test/resources/hbase/hbase-site.xml b/pulsar-io/hbase/src/test/resources/hbase/hbase-site.xml index c2b520a765643..460cbd497c15f 100644 --- a/pulsar-io/hbase/src/test/resources/hbase/hbase-site.xml +++ b/pulsar-io/hbase/src/test/resources/hbase/hbase-site.xml @@ -1,3 +1,4 @@ + - - hbase.cluster.distributed - true - - - hbase.rootdir - hdfs://localhost:8020/hbase - - - hbase.zookeeper.quorum - localhost - - - hbase.zookeeper.property.clientPort - 2181 - - - zookeeper.znode.parent - /hbase - + + hbase.cluster.distributed + true + + + hbase.rootdir + hdfs://localhost:8020/hbase + + + hbase.zookeeper.quorum + localhost + + + hbase.zookeeper.property.clientPort + 2181 + + + zookeeper.znode.parent + /hbase + diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml index d800a87960169..b642c12623b2f 100644 --- a/pulsar-io/hdfs2/pom.xml +++ b/pulsar-io/hdfs2/pom.xml @@ -1,3 +1,4 @@ + - - owasp-dependency-check - - - - org.owasp - dependency-check-maven - ${dependency-check-maven.version} - - - - aggregate - - none - - - - - - - - - \ No newline at end of file + + owasp-dependency-check + + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + + + aggregate + + none + + + + + + + + diff --git a/pulsar-io/hdfs2/src/main/resources/findbugsExclude.xml b/pulsar-io/hdfs2/src/main/resources/findbugsExclude.xml index 980349c4d8d5e..93756f7f28461 100644 --- a/pulsar-io/hdfs2/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/hdfs2/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - fs.defaultFS - hdfs://0.0.0.0:8020 - - - io.compression.codecs - org.apache.hadoop.io.compress.GzipCodec, + + fs.defaultFS + hdfs://0.0.0.0:8020 + + + io.compression.codecs + org.apache.hadoop.io.compress.GzipCodec, org.apache.hadoop.io.compress.DefaultCodec, org.apache.hadoop.io.compress.SnappyCodec - + diff --git a/pulsar-io/hdfs2/src/test/resources/hadoop/hdfs-site.xml b/pulsar-io/hdfs2/src/test/resources/hadoop/hdfs-site.xml index bb722f1f63470..1b35ebd585012 100644 --- a/pulsar-io/hdfs2/src/test/resources/hadoop/hdfs-site.xml +++ b/pulsar-io/hdfs2/src/test/resources/hadoop/hdfs-site.xml @@ -1,3 +1,4 @@ + - - dfs.replication - 1 - - - dfs.client.use.datanode.hostname - true - - - dfs.support.append - true - + + dfs.replication + 1 + + + dfs.client.use.datanode.hostname + true + + + dfs.support.append + true + diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml index e52f7a68f602b..e8bc927f4fe53 100644 --- a/pulsar-io/hdfs3/pom.xml +++ b/pulsar-io/hdfs3/pom.xml @@ -1,3 +1,4 @@ + - - fs.defaultFS - hdfs://0.0.0.0:8020 - - - io.compression.codecs - org.apache.hadoop.io.compress.GzipCodec, + + fs.defaultFS + hdfs://0.0.0.0:8020 + + + io.compression.codecs + org.apache.hadoop.io.compress.GzipCodec, org.apache.hadoop.io.compress.DefaultCodec, org.apache.hadoop.io.compress.SnappyCodec - + diff --git a/pulsar-io/hdfs3/src/test/resources/hadoop/hdfs-site.xml b/pulsar-io/hdfs3/src/test/resources/hadoop/hdfs-site.xml index bb722f1f63470..1b35ebd585012 100644 --- a/pulsar-io/hdfs3/src/test/resources/hadoop/hdfs-site.xml +++ b/pulsar-io/hdfs3/src/test/resources/hadoop/hdfs-site.xml @@ -1,3 +1,4 @@ + - - dfs.replication - 1 - - - dfs.client.use.datanode.hostname - true - - - dfs.support.append - true - + + dfs.replication + 1 + + + dfs.client.use.datanode.hostname + true + + + dfs.support.append + true + diff --git a/pulsar-io/http/pom.xml b/pulsar-io/http/pom.xml index 86d1290774cd5..0388d08e8a06b 100644 --- a/pulsar-io/http/pom.xml +++ b/pulsar-io/http/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io - 3.2.0-SNAPSHOT - - - pulsar-io-http - Pulsar IO :: HTTP - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - com.fasterxml.jackson.datatype - jackson-datatype-jsr310 - - - - org.apache.avro - avro - ${avro.version} - - - - org.apache.pulsar - pulsar-client-original - ${project.version} - test - - - - com.github.tomakehurst - wiremock-jre8 - ${wiremock.version} - test - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - + + 4.0.0 + + io.streamnative + pulsar-io + 3.2.0-SNAPSHOT + + pulsar-io-http + Pulsar IO :: HTTP + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + + + org.apache.avro + avro + ${avro.version} + + + io.streamnative + pulsar-client-original + ${project.version} + test + + + com.github.tomakehurst + wiremock-jre8 + ${wiremock.version} + test + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + diff --git a/pulsar-io/influxdb/pom.xml b/pulsar-io/influxdb/pom.xml index 5f7e550989ef3..d68fd8210892c 100644 --- a/pulsar-io/influxdb/pom.xml +++ b/pulsar-io/influxdb/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - pulsar-io - org.apache.pulsar - 3.2.0-SNAPSHOT - - - pulsar-io-influxdb - Pulsar IO :: InfluxDB - - - - ${project.groupId} - pulsar-io-common - ${project.version} - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - ${project.groupId} - pulsar-functions-instance - ${project.version} - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - - com.influxdb - influxdb-client-java - 4.0.0 - - - - org.influxdb - influxdb-java - 2.22 - - - com.squareup.okhttp3 - * - - - - - - org.apache.commons - commons-lang3 - - - org.apache.commons - commons-collections4 - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - - \ No newline at end of file + + 4.0.0 + + pulsar-io + io.streamnative + 3.2.0-SNAPSHOT + + pulsar-io-influxdb + Pulsar IO :: InfluxDB + + + ${project.groupId} + pulsar-io-common + ${project.version} + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + ${project.groupId} + pulsar-functions-instance + ${project.version} + + + ${project.groupId} + pulsar-client-original + ${project.version} + + + com.influxdb + influxdb-client-java + 4.0.0 + + + org.influxdb + influxdb-java + 2.22 + + + com.squareup.okhttp3 + * + + + + + org.apache.commons + commons-lang3 + + + org.apache.commons + commons-collections4 + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + + diff --git a/pulsar-io/jdbc/clickhouse/pom.xml b/pulsar-io/jdbc/clickhouse/pom.xml index 81033e6499776..7e82f043b256f 100644 --- a/pulsar-io/jdbc/clickhouse/pom.xml +++ b/pulsar-io/jdbc/clickhouse/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar + io.streamnative 3.2.0-SNAPSHOT 4.0.0 - pulsar-io-jdbc-clickhouse Pulsar IO :: Jdbc :: ClickHouse - ${project.groupId} @@ -45,13 +42,12 @@ all - * - * + * + * - diff --git a/pulsar-io/jdbc/core/pom.xml b/pulsar-io/jdbc/core/pom.xml index ae4e4f77d5386..04eeda522e450 100644 --- a/pulsar-io/jdbc/core/pom.xml +++ b/pulsar-io/jdbc/core/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar + io.streamnative 3.2.0-SNAPSHOT 4.0.0 - pulsar-io-jdbc-core Pulsar IO :: Jdbc :: Core - ${project.groupId} pulsar-io-common ${project.version} - ${project.groupId} pulsar-io-core ${project.version} - org.apache.avro avro ${avro.version} - com.fasterxml.jackson.core jackson-databind - com.google.guava guava - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.google.protobuf protobuf-java provided - - - \ No newline at end of file + diff --git a/pulsar-io/jdbc/mariadb/pom.xml b/pulsar-io/jdbc/mariadb/pom.xml index 06079ab94c7d3..0e516a742950c 100644 --- a/pulsar-io/jdbc/mariadb/pom.xml +++ b/pulsar-io/jdbc/mariadb/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar + io.streamnative 3.2.0-SNAPSHOT 4.0.0 - pulsar-io-jdbc-mariadb Pulsar IO :: Jdbc :: Mariadb - ${project.groupId} @@ -52,4 +49,4 @@ - \ No newline at end of file + diff --git a/pulsar-io/jdbc/openmldb/pom.xml b/pulsar-io/jdbc/openmldb/pom.xml index 897fd63ae90dc..0e919c779b555 100644 --- a/pulsar-io/jdbc/openmldb/pom.xml +++ b/pulsar-io/jdbc/openmldb/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar + io.streamnative 3.2.0-SNAPSHOT 4.0.0 - pulsar-io-jdbc-openmldb Pulsar IO :: Jdbc :: OpenMLDB - ${project.groupId} @@ -60,7 +57,6 @@ runtime - diff --git a/pulsar-io/jdbc/pom.xml b/pulsar-io/jdbc/pom.xml index 25416dc5b47a6..c8f7af8a247fa 100644 --- a/pulsar-io/jdbc/pom.xml +++ b/pulsar-io/jdbc/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom @@ -31,12 +31,10 @@ openmldb - org.apache.pulsar + io.streamnative pulsar-io 3.2.0-SNAPSHOT - pulsar-io-jdbc Pulsar IO :: Jdbc - diff --git a/pulsar-io/jdbc/postgres/pom.xml b/pulsar-io/jdbc/postgres/pom.xml index 8b489f509ed33..b6d9a3d04b514 100644 --- a/pulsar-io/jdbc/postgres/pom.xml +++ b/pulsar-io/jdbc/postgres/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar + io.streamnative 3.2.0-SNAPSHOT 4.0.0 - pulsar-io-jdbc-postgres Pulsar IO :: Jdbc :: Postgres - ${project.groupId} @@ -44,7 +41,6 @@ runtime - diff --git a/pulsar-io/jdbc/sqlite/pom.xml b/pulsar-io/jdbc/sqlite/pom.xml index b1ffdb93608ab..fb05ddb16fb4e 100644 --- a/pulsar-io/jdbc/sqlite/pom.xml +++ b/pulsar-io/jdbc/sqlite/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar + io.streamnative 3.2.0-SNAPSHOT 4.0.0 pulsar-io-jdbc-sqlite Pulsar IO :: Jdbc :: Sqlite - ${project.groupId} @@ -60,7 +58,6 @@ test - @@ -69,4 +66,4 @@ - \ No newline at end of file + diff --git a/pulsar-io/kafka-connect-adaptor-nar/pom.xml b/pulsar-io/kafka-connect-adaptor-nar/pom.xml index 1d3459dfb6c3b..d1eecd5200670 100644 --- a/pulsar-io/kafka-connect-adaptor-nar/pom.xml +++ b/pulsar-io/kafka-connect-adaptor-nar/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.2.0-SNAPSHOT - pulsar-io-kafka-connect-adaptor-nar Pulsar IO :: Kafka Connect Adaptor NAR - ${project.groupId} @@ -37,8 +35,6 @@ ${project.version} - - @@ -48,7 +44,30 @@ pulsar-io-kafka-connect-adaptor-${project.version} + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + - diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index fefad7dbe20ee..e27d942824266 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.2.0-SNAPSHOT - pulsar-io-kafka-connect-adaptor Pulsar IO :: Kafka Connect Adaptor - - ${project.groupId} pulsar-io-core ${project.version} provided - ${project.groupId} pulsar-common ${project.version} compile - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - org.apache.kafka connect-runtime @@ -79,7 +72,6 @@ - org.apache.kafka connect-json @@ -91,7 +83,6 @@ - org.apache.kafka connect-api @@ -103,7 +94,6 @@ - ${project.groupId} @@ -116,17 +106,14 @@ - io.netty netty-buffer - org.apache.commons commons-lang3 - io.confluent @@ -139,14 +126,12 @@ - ${project.groupId} pulsar-broker ${project.version} test - org.apache.kafka connect-file @@ -159,14 +144,12 @@ - ${project.groupId} testmocks ${project.version} test - ${project.groupId} pulsar-broker @@ -174,38 +157,37 @@ test test-jar - org.apache.avro avro ${avro.version} provided - com.google.protobuf protobuf-java provided - org.asynchttpclient async-http-client test - org.bouncycastle bc-fips ${bouncycastle.bc-fips.version} test - com.typesafe.netty netty-reactive-streams test + + com.google.protobuf + protobuf-java + 3.5.1 + - diff --git a/pulsar-io/kafka/pom.xml b/pulsar-io/kafka/pom.xml index 33268854151b1..21569638f0964 100644 --- a/pulsar-io/kafka/pom.xml +++ b/pulsar-io/kafka/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.2.0-SNAPSHOT - pulsar-io-kafka Pulsar IO :: Kafka - @@ -44,42 +42,35 @@ - ${project.groupId} pulsar-io-common ${project.version} - ${project.groupId} pulsar-io-core ${project.version} provided - ${project.groupId} pulsar-client-original ${project.version} - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.google.guava guava - org.apache.kafka kafka-clients @@ -91,43 +82,35 @@ - io.confluent kafka-schema-registry-client ${confluent.version} - io.confluent kafka-avro-serializer ${confluent.version} - io.jsonwebtoken jjwt-impl - io.jsonwebtoken jjwt-jackson - org.hamcrest hamcrest test - org.awaitility awaitility test - - diff --git a/pulsar-io/kafka/src/main/resources/findbugsExclude.xml b/pulsar-io/kafka/src/main/resources/findbugsExclude.xml index 3fc8f67c8640e..a48959506a69a 100644 --- a/pulsar-io/kafka/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/kafka/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.2.0-SNAPSHOT - pulsar-io-kinesis Pulsar IO :: Kinesis - 2.2.8 0.14.0 @@ -37,116 +35,96 @@ 1.9.0 2.3.0 - ${project.groupId} pulsar-io-common ${project.version} - ${project.groupId} pulsar-io-core ${project.version} - ${project.groupId} pulsar-io-aws ${project.version} - - org.apache.pulsar + io.streamnative pulsar-functions-instance ${project.version} test - org.apache.commons commons-lang3 - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.fasterxml.jackson.dataformat jackson-dataformat-cbor - org.apache.avro avro ${avro.version} - com.github.wnameless.json json-flattener ${json-flattener.version} - com.google.code.gson gson - com.amazonaws aws-java-sdk-core - software.amazon.kinesis amazon-kinesis-client ${amazon-kinesis-client.version} - com.amazonaws amazon-kinesis-producer ${amazon-kinesis-producer.version} - - + com.google.flatbuffers flatbuffers-java ${flatbuffers-java.version} - javax.xml.bind jaxb-api ${jaxb-api.version} - org.testcontainers localstack test - org.awaitility awaitility test - - @@ -155,5 +133,4 @@ - diff --git a/pulsar-io/mongo/pom.xml b/pulsar-io/mongo/pom.xml index 7ad363f3b381a..08f566c1844f2 100644 --- a/pulsar-io/mongo/pom.xml +++ b/pulsar-io/mongo/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar - pulsar-io - 3.2.0-SNAPSHOT + io.streamnative + pulsar-io + 3.2.0-SNAPSHOT - pulsar-io-mongo Pulsar IO :: MongoDB - 4.1.2 - ${project.groupId} @@ -69,7 +64,6 @@ guava - diff --git a/pulsar-io/netty/pom.xml b/pulsar-io/netty/pom.xml index 57b0694bdbf55..f4d6dfdcfae11 100644 --- a/pulsar-io/netty/pom.xml +++ b/pulsar-io/netty/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io - 3.2.0-SNAPSHOT - - - pulsar-io-netty - Pulsar IO :: Netty - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - io.netty - netty-handler - - - - io.netty - netty-codec-http - - - - com.google.guava - guava - - - - org.apache.commons - commons-lang3 - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - spotbugs - verify - - check - - - - - - - + + 4.0.0 + + io.streamnative + pulsar-io + 3.2.0-SNAPSHOT + + pulsar-io-netty + Pulsar IO :: Netty + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + io.netty + netty-handler + + + io.netty + netty-codec-http + + + com.google.guava + guava + + + org.apache.commons + commons-lang3 + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + spotbugs + verify + + check + + + + + + diff --git a/pulsar-io/netty/src/main/resources/findbugsExclude.xml b/pulsar-io/netty/src/main/resources/findbugsExclude.xml index 3a52062fc9233..6b8906a775bd3 100644 --- a/pulsar-io/netty/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/netty/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - - - - + + + + + + + + + + + + diff --git a/pulsar-io/nsq/pom.xml b/pulsar-io/nsq/pom.xml index 4da0558e87d96..be9e26cde1a15 100644 --- a/pulsar-io/nsq/pom.xml +++ b/pulsar-io/nsq/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.2.0-SNAPSHOT - pulsar-io-nsq Pulsar IO :: NSQ - ${project.groupId} pulsar-io-core ${project.version} - com.sproutsocial nsq-j ${nsq-client.version} - org.apache.commons commons-collections4 - org.apache.commons commons-lang3 - ${project.groupId} pulsar-io-common ${project.version} - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - - - - - org.apache.nifi - nifi-nar-maven-plugin - - + + + org.apache.nifi + nifi-nar-maven-plugin + + diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml index 201ca1f5155ec..d46387c1ea4bc 100644 --- a/pulsar-io/pom.xml +++ b/pulsar-io/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT - pulsar-io Pulsar IO :: Parent - main @@ -77,7 +75,6 @@ alluxio - pulsar-io-tests @@ -110,7 +107,6 @@ alluxio - pulsar-io-elastic-tests @@ -119,7 +115,6 @@ elastic-search - pulsar-io-kafka-connect-tests @@ -131,7 +126,6 @@ debezium - core-modules @@ -145,7 +139,6 @@ - @@ -163,5 +156,4 @@ - diff --git a/pulsar-io/rabbitmq/pom.xml b/pulsar-io/rabbitmq/pom.xml index 33f3dc55b891c..e3ea823f15b1c 100644 --- a/pulsar-io/rabbitmq/pom.xml +++ b/pulsar-io/rabbitmq/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.2.0-SNAPSHOT - pulsar-io-rabbitmq Pulsar IO :: RabbitMQ - - ${project.groupId} pulsar-io-common @@ -58,28 +55,23 @@ pulsar-client-original ${project.version} - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.google.guava guava - com.rabbitmq amqp-client ${rabbitmq-client.version} - org.apache.qpid qpid-broker @@ -91,9 +83,7 @@ awaitility test - - @@ -102,5 +92,4 @@ - diff --git a/pulsar-io/redis/pom.xml b/pulsar-io/redis/pom.xml index 67560cc41b451..fb45c1b75bcec 100644 --- a/pulsar-io/redis/pom.xml +++ b/pulsar-io/redis/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - pulsar-io - org.apache.pulsar - 3.2.0-SNAPSHOT - - - pulsar-io-redis - Pulsar IO :: Redis - - - - ${project.groupId} - pulsar-io-common - ${project.version} - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - ${project.groupId} - pulsar-functions-instance - ${project.version} - - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - io.lettuce - lettuce-core - 5.0.2.RELEASE - - - com.google.guava - guava - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - org.apache.commons - commons-lang3 - - - org.apache.commons - commons-collections4 - - - com.github.kstyrc - embedded-redis - 0.6 - test - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - \ No newline at end of file + + 4.0.0 + + pulsar-io + io.streamnative + 3.2.0-SNAPSHOT + + pulsar-io-redis + Pulsar IO :: Redis + + + ${project.groupId} + pulsar-io-common + ${project.version} + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + ${project.groupId} + pulsar-functions-instance + ${project.version} + + + ${project.groupId} + pulsar-client-original + ${project.version} + + + io.lettuce + lettuce-core + 5.0.2.RELEASE + + + com.google.guava + guava + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + org.apache.commons + commons-lang3 + + + org.apache.commons + commons-collections4 + + + com.github.kstyrc + embedded-redis + 0.6 + test + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + + diff --git a/pulsar-io/solr/pom.xml b/pulsar-io/solr/pom.xml index 249357dd3bf28..1c792c0b4ae15 100644 --- a/pulsar-io/solr/pom.xml +++ b/pulsar-io/solr/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - pulsar-io - org.apache.pulsar - 3.2.0-SNAPSHOT - - - - 8.11.1 - - - pulsar-io-solr - Pulsar IO :: Solr - - - - ${project.groupId} - pulsar-io-common - ${project.version} - - - ${project.parent.groupId} - pulsar-io-core - ${project.parent.version} - - - ${project.groupId} - pulsar-functions-instance - ${project.version} - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - org.apache.solr - solr-solrj - ${solr.version} - - - org.apache.solr - solr-core - ${solr.version} - - - jose4j - org.bitbucket.b_c - - - test - - - org.apache.commons - commons-lang3 - - - org.apache.commons - commons-collections4 - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - com.github.spotbugs - spotbugs-maven-plugin - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - spotbugs - verify - - check - - - - - - - - \ No newline at end of file + + 4.0.0 + + pulsar-io + io.streamnative + 3.2.0-SNAPSHOT + + + 8.11.3 + + pulsar-io-solr + Pulsar IO :: Solr + + + ${project.groupId} + pulsar-io-common + ${project.version} + + + ${project.parent.groupId} + pulsar-io-core + ${project.parent.version} + + + ${project.groupId} + pulsar-functions-instance + ${project.version} + + + ${project.groupId} + pulsar-client-original + ${project.version} + + + org.apache.solr + solr-solrj + ${solr.version} + + + org.apache.solr + solr-core + ${solr.version} + + + jose4j + org.bitbucket.b_c + + + test + + + org.apache.commons + commons-lang3 + + + org.apache.commons + commons-collections4 + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + com.github.spotbugs + spotbugs-maven-plugin + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + spotbugs + verify + + check + + + + + + + diff --git a/pulsar-io/solr/src/main/resources/findbugsExclude.xml b/pulsar-io/solr/src/main/resources/findbugsExclude.xml index f49ff86c90caf..47c3d73af5f06 100644 --- a/pulsar-io/solr/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/solr/src/main/resources/findbugsExclude.xml @@ -1,22 +1,23 @@ - - - \ No newline at end of file + + + + diff --git a/pulsar-io/solr/src/test/resources/solr.xml b/pulsar-io/solr/src/test/resources/solr.xml index 495a71f72f3f2..9ce4d37f4beef 100644 --- a/pulsar-io/solr/src/test/resources/solr.xml +++ b/pulsar-io/solr/src/test/resources/solr.xml @@ -1,3 +1,4 @@ + - - ${host:} - ${jetty.port:8983} - ${hostContext:solr} - ${genericCoreNodeNames:true} - ${zkClientTimeout:30000} - ${distribUpdateSoTimeout:600000} - ${distribUpdateConnTimeout:60000} - ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} - ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} - - - ${socketTimeout:600000} - ${connTimeout:60000} - - \ No newline at end of file + + ${host:} + ${jetty.port:8983} + ${hostContext:solr} + ${genericCoreNodeNames:true} + ${zkClientTimeout:30000} + ${distribUpdateSoTimeout:600000} + ${distribUpdateConnTimeout:60000} + ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} + ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} + + + ${socketTimeout:600000} + ${connTimeout:60000} + + diff --git a/pulsar-io/twitter/pom.xml b/pulsar-io/twitter/pom.xml index 2cab4de5d3ab5..15b212bab3e9c 100644 --- a/pulsar-io/twitter/pom.xml +++ b/pulsar-io/twitter/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io 3.2.0-SNAPSHOT - pulsar-io-twitter Pulsar IO :: Twitter - - ${project.groupId} pulsar-io-core ${project.version} - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.twitter hbc-core ${hbc-core.version} - org.apache.commons commons-collections4 - org.apache.commons commons-lang3 - - ${project.groupId} - pulsar-io-common - ${project.version} + ${project.groupId} + pulsar-io-common + ${project.version} - - @@ -80,5 +69,4 @@ - diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index 9403a5c0027d4..938a72220f482 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - 2 - pulsar-metadata Pulsar Metadata - org.apache.pulsar + io.streamnative pulsar-common ${project.version} - org.apache.zookeeper zookeeper @@ -61,13 +57,11 @@ - io.dropwizard.metrics metrics-core test - org.apache.zookeeper zookeeper @@ -99,42 +93,33 @@ snappy-java test - org.awaitility awaitility test - org.apache.bookkeeper bookkeeper-server - io.etcd jetcd-core - - io.etcd jetcd-test test - com.github.ben-manes.caffeine caffeine - io.prometheus simpleclient - - @@ -154,7 +139,6 @@ - org.apache.maven.plugins maven-jar-plugin diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java index 9ba2588a07cf0..0a35664391455 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java @@ -177,6 +177,14 @@ public CompletableFuture handleMetadataEvent(MetadataEvent event) { return result; } + /** + * @deprecated Use {@link #registerSyncListener(Optional)} instead. + */ + @Deprecated + protected void registerSyncLister(Optional synchronizer) { + this.registerSyncListener(synchronizer); + } + protected void registerSyncListener(Optional synchronizer) { synchronizer.ifPresent(s -> s.registerSyncListener(this::handleMetadataEvent)); } diff --git a/pulsar-metadata/src/main/resources/findbugsExclude.xml b/pulsar-metadata/src/main/resources/findbugsExclude.xml index 21578b9f8cb5c..d6eb8623eb73e 100644 --- a/pulsar-metadata/src/main/resources/findbugsExclude.xml +++ b/pulsar-metadata/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - - - - + + + + + + + + + + + + diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java index a1954831abf3f..220b2ed179972 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java @@ -42,6 +42,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import lombok.Cleanup; import org.apache.bookkeeper.bookie.BookieImpl; @@ -425,8 +426,16 @@ public void testInnerDelayedAuditOfLostBookies() throws Exception { // wait for 5 seconds before starting the recovery work when a bookie fails urLedgerMgr.setLostBookieRecoveryDelay(5); - // shutdown a non auditor bookie; choosing non-auditor to avoid another election - String shutdownBookie = shutDownNonAuditorBookie(); + AtomicReference shutdownBookieRef = new AtomicReference<>(); + CountDownLatch shutdownLatch = new CountDownLatch(1); + new Thread(() -> { + try { + String shutdownBookie = shutDownNonAuditorBookie(); + shutdownBookieRef.set(shutdownBookie); + shutdownLatch.countDown(); + } catch (Exception ignore) { + } + }).start(); if (LOG.isDebugEnabled()) { LOG.debug("Waiting for ledgers to be marked as under replicated"); @@ -442,9 +451,10 @@ public void testInnerDelayedAuditOfLostBookies() throws Exception { urLedgerList.contains(ledgerId)); Map urLedgerData = getUrLedgerData(urLedgerList); String data = urLedgerData.get(ledgerId); - assertTrue("Bookie " + shutdownBookie + shutdownLatch.await(); + assertTrue("Bookie " + shutdownBookieRef.get() + "is not listed in the ledger as missing replica :" + data, - data.contains(shutdownBookie)); + data.contains(shutdownBookieRef.get())); } /** @@ -503,7 +513,16 @@ public void testRescheduleOfDelayedAuditOfLostBookiesToStartImmediately() throws urLedgerMgr.setLostBookieRecoveryDelay(50); // shutdown a non auditor bookie; choosing non-auditor to avoid another election - String shutdownBookie = shutDownNonAuditorBookie(); + AtomicReference shutdownBookieRef = new AtomicReference<>(); + CountDownLatch shutdownLatch = new CountDownLatch(1); + new Thread(() -> { + try { + String shutdownBookie = shutDownNonAuditorBookie(); + shutdownBookieRef.set(shutdownBookie); + shutdownLatch.countDown(); + } catch (Exception ignore) { + } + }).start(); if (LOG.isDebugEnabled()) { LOG.debug("Waiting for ledgers to be marked as under replicated"); @@ -522,9 +541,10 @@ public void testRescheduleOfDelayedAuditOfLostBookiesToStartImmediately() throws urLedgerList.contains(ledgerId)); Map urLedgerData = getUrLedgerData(urLedgerList); String data = urLedgerData.get(ledgerId); - assertTrue("Bookie " + shutdownBookie + shutdownLatch.await(); + assertTrue("Bookie " + shutdownBookieRef.get() + "is not listed in the ledger as missing replica :" + data, - data.contains(shutdownBookie)); + data.contains(shutdownBookieRef.get())); } @Test @@ -547,7 +567,16 @@ public void testRescheduleOfDelayedAuditOfLostBookiesToStartLater() throws Excep urLedgerMgr.setLostBookieRecoveryDelay(3); // shutdown a non auditor bookie; choosing non-auditor to avoid another election - String shutdownBookie = shutDownNonAuditorBookie(); + AtomicReference shutdownBookieRef = new AtomicReference<>(); + CountDownLatch shutdownLatch = new CountDownLatch(1); + new Thread(() -> { + try { + String shutdownBookie = shutDownNonAuditorBookie(); + shutdownBookieRef.set(shutdownBookie); + shutdownLatch.countDown(); + } catch (Exception ignore) { + } + }).start(); if (LOG.isDebugEnabled()) { LOG.debug("Waiting for ledgers to be marked as under replicated"); @@ -573,9 +602,10 @@ public void testRescheduleOfDelayedAuditOfLostBookiesToStartLater() throws Excep urLedgerList.contains(ledgerId)); Map urLedgerData = getUrLedgerData(urLedgerList); String data = urLedgerData.get(ledgerId); - assertTrue("Bookie " + shutdownBookie + shutdownLatch.await(); + assertTrue("Bookie " + shutdownBookieRef.get() + "is not listed in the ledger as missing replica :" + data, - data.contains(shutdownBookie)); + data.contains(shutdownBookieRef.get())); } @Test @@ -664,7 +694,12 @@ public void testTriggerAuditorWithPendingAuditTask() throws Exception { urLedgerMgr.setLostBookieRecoveryDelay(lostBookieRecoveryDelay); // shutdown a non auditor bookie; choosing non-auditor to avoid another election - String shutdownBookie = shutDownNonAuditorBookie(); + new Thread(() -> { + try { + shutDownNonAuditorBookie(); + } catch (Exception ignore) { + } + }).start(); if (LOG.isDebugEnabled()) { LOG.debug("Waiting for ledgers to be marked as under replicated"); @@ -715,7 +750,12 @@ public void testTriggerAuditorBySettingDelayToZeroWithPendingAuditTask() throws urLedgerMgr.setLostBookieRecoveryDelay(lostBookieRecoveryDelay); // shutdown a non auditor bookie; choosing non-auditor to avoid another election - String shutdownBookie = shutDownNonAuditorBookie(); + new Thread(() -> { + try { + shutDownNonAuditorBookie(); + } catch (Exception ignore) { + } + }).start(); if (LOG.isDebugEnabled()) { LOG.debug("Waiting for ledgers to be marked as under replicated"); @@ -767,8 +807,17 @@ public void testDelayedAuditWithMultipleBookieFailures() throws Exception { // wait for 10 seconds before starting the recovery work when a bookie fails urLedgerMgr.setLostBookieRecoveryDelay(10); - // shutdown a non auditor bookie to avoid an election - String shutdownBookie1 = shutDownNonAuditorBookie(); + // shutdown a non auditor bookie; choosing non-auditor to avoid another election + AtomicReference shutdownBookieRef1 = new AtomicReference<>(); + CountDownLatch shutdownLatch1 = new CountDownLatch(1); + new Thread(() -> { + try { + String shutdownBookie1 = shutDownNonAuditorBookie(); + shutdownBookieRef1.set(shutdownBookie1); + shutdownLatch1.countDown(); + } catch (Exception ignore) { + } + }).start(); // wait for 3 seconds and there shouldn't be any under replicated ledgers // because we have delayed the start of audit by 10 seconds @@ -780,7 +829,16 @@ public void testDelayedAuditWithMultipleBookieFailures() throws Exception { // the history about having delayed recovery remains. Hence we make sure // we bring down a non auditor bookie. This should cause the audit to take // place immediately and not wait for the remaining 7 seconds to elapse - String shutdownBookie2 = shutDownNonAuditorBookie(); + AtomicReference shutdownBookieRef2 = new AtomicReference<>(); + CountDownLatch shutdownLatch2 = new CountDownLatch(1); + new Thread(() -> { + try { + String shutdownBookie2 = shutDownNonAuditorBookie(); + shutdownBookieRef2.set(shutdownBookie2); + shutdownLatch2.countDown(); + } catch (Exception ignore) { + } + }).start(); // 2 second grace period for the ledgers to get reported as under replicated Thread.sleep(2000); @@ -793,9 +851,11 @@ public void testDelayedAuditWithMultipleBookieFailures() throws Exception { urLedgerList.contains(ledgerId)); Map urLedgerData = getUrLedgerData(urLedgerList); String data = urLedgerData.get(ledgerId); - assertTrue("Bookie " + shutdownBookie1 + shutdownBookie2 + shutdownLatch1.await(); + shutdownLatch2.await(); + assertTrue("Bookie " + shutdownBookieRef1.get() + shutdownBookieRef2.get() + " are not listed in the ledger as missing replicas :" + data, - data.contains(shutdownBookie1) && data.contains(shutdownBookie2)); + data.contains(shutdownBookieRef1.get()) && data.contains(shutdownBookieRef2.get())); } /** @@ -825,7 +885,16 @@ public void testDelayedAuditWithRollingUpgrade() throws Exception { // shutdown a non auditor bookie to avoid an election int idx1 = getShutDownNonAuditorBookieIdx(""); ServerConfiguration conf1 = confByIndex(idx1); - String shutdownBookie1 = shutdownBookie(idx1); + AtomicReference shutdownBookieRef1 = new AtomicReference<>(); + CountDownLatch shutdownLatch1 = new CountDownLatch(1); + new Thread(() -> { + try { + String shutdownBookie1 = shutdownBookie(idx1); + shutdownBookieRef1.set(shutdownBookie1); + shutdownLatch1.countDown(); + } catch (Exception ignore) { + } + }).start(); // wait for 2 seconds and there shouldn't be any under replicated ledgers // because we have delayed the start of audit by 5 seconds @@ -838,7 +907,17 @@ public void testDelayedAuditWithRollingUpgrade() throws Exception { // Now to simulate the rolling upgrade, bring down a bookie different from // the one we brought down/up above. - String shutdownBookie2 = shutDownNonAuditorBookie(shutdownBookie1); + // shutdown a non auditor bookie; choosing non-auditor to avoid another election + AtomicReference shutdownBookieRef2 = new AtomicReference<>(); + CountDownLatch shutdownLatch2 = new CountDownLatch(1); + new Thread(() -> { + try { + String shutdownBookie2 = shutDownNonAuditorBookie(); + shutdownBookieRef2.set(shutdownBookie2); + shutdownLatch2.countDown(); + } catch (Exception ignore) { + } + }).start(); // since the first bookie that was brought down/up has come up, there is only // one bookie down at this time. Hence the lost bookie check shouldn't start @@ -856,11 +935,13 @@ public void testDelayedAuditWithRollingUpgrade() throws Exception { urLedgerList.contains(ledgerId)); Map urLedgerData = getUrLedgerData(urLedgerList); String data = urLedgerData.get(ledgerId); - assertTrue("Bookie " + shutdownBookie1 + "wrongly listed as missing the ledger: " + data, - !data.contains(shutdownBookie1)); - assertTrue("Bookie " + shutdownBookie2 + shutdownLatch1.await(); + shutdownLatch2.await(); + assertTrue("Bookie " + shutdownBookieRef1.get() + "wrongly listed as missing the ledger: " + data, + !data.contains(shutdownBookieRef1.get())); + assertTrue("Bookie " + shutdownBookieRef2.get() + " is not listed in the ledger as missing replicas :" + data, - data.contains(shutdownBookie2)); + data.contains(shutdownBookieRef2.get())); LOG.info("*****************Test Complete"); } diff --git a/pulsar-metadata/src/test/resources/findbugsExclude.xml b/pulsar-metadata/src/test/resources/findbugsExclude.xml index e40b2a0fd9a12..bf075c39791cf 100644 --- a/pulsar-metadata/src/test/resources/findbugsExclude.xml +++ b/pulsar-metadata/src/test/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - \ No newline at end of file + + + + + + + + + diff --git a/pulsar-package-management/bookkeeper-storage/pom.xml b/pulsar-package-management/bookkeeper-storage/pom.xml index bd1d1f4ca378f..6899f2dffc291 100644 --- a/pulsar-package-management/bookkeeper-storage/pom.xml +++ b/pulsar-package-management/bookkeeper-storage/pom.xml @@ -1,4 +1,4 @@ - + - - - pulsar-package-management - org.apache.pulsar - 3.2.0-SNAPSHOT - - 4.0.0 - - pulsar-package-bookkeeper-storage - Apache Pulsar :: Package Management :: BookKeeper Storage - - - - ${project.groupId} - pulsar-package-core - ${project.version} - - - - org.apache.distributedlog - distributedlog-core - - - net.jpountz.lz4 - lz4 - - - - - - org.apache.zookeeper - zookeeper - tests - test - - - ch.qos.logback - logback-core - - - ch.qos.logback - logback-classic - - - io.netty - netty-tcnative - - - - - - org.hamcrest - hamcrest - test - - - - - io.dropwizard.metrics - metrics-core - test - - - - - org.xerial.snappy - snappy-java - test - - - - ${project.groupId} - managed-ledger - ${project.version} - test - - - - ${project.groupId} - testmocks - ${project.version} - - - + + + pulsar-package-management + io.streamnative + 3.2.0-SNAPSHOT + + 4.0.0 + pulsar-package-bookkeeper-storage + Apache Pulsar :: Package Management :: BookKeeper Storage + + + ${project.groupId} + pulsar-package-core + ${project.version} + + + org.apache.distributedlog + distributedlog-core + + + net.jpountz.lz4 + lz4 + + + + + org.apache.zookeeper + zookeeper + tests + test + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + io.netty + netty-tcnative + + + + + org.hamcrest + hamcrest + test + + + + io.dropwizard.metrics + metrics-core + test + + + + org.xerial.snappy + snappy-java + test + + + ${project.groupId} + managed-ledger + ${project.version} + test + + + ${project.groupId} + testmocks + ${project.version} + + diff --git a/pulsar-package-management/core/pom.xml b/pulsar-package-management/core/pom.xml index 05d8684bb8634..0ba6820064dcc 100644 --- a/pulsar-package-management/core/pom.xml +++ b/pulsar-package-management/core/pom.xml @@ -1,4 +1,4 @@ - + - - - pulsar-package-management - org.apache.pulsar - 3.2.0-SNAPSHOT - - 4.0.0 - - pulsar-package-core - Apache Pulsar :: Package Management :: Core - - - - ${project.groupId} - pulsar-client-admin-api - ${project.parent.version} - - - - com.google.guava - guava - - - - org.apache.commons - commons-lang3 - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - - ${pulsar.client.compiler.release} - - - - org.gaul - modernizer-maven-plugin - - true - 17 - - - - modernizer - verify - - modernizer - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - + + + pulsar-package-management + io.streamnative + 3.2.0-SNAPSHOT + + 4.0.0 + pulsar-package-core + Apache Pulsar :: Package Management :: Core + + + ${project.groupId} + pulsar-client-admin-api + ${project.parent.version} + + + com.google.guava + guava + + + org.apache.commons + commons-lang3 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + ${pulsar.client.compiler.release} + + + + org.gaul + modernizer-maven-plugin + + true + 17 + + + + modernizer + verify + + modernizer + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + diff --git a/pulsar-package-management/core/src/main/resources/findbugsExclude.xml b/pulsar-package-management/core/src/main/resources/findbugsExclude.xml index 3a2e998dce984..bd8d2c4f77efa 100644 --- a/pulsar-package-management/core/src/main/resources/findbugsExclude.xml +++ b/pulsar-package-management/core/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - pulsar-package-management - org.apache.pulsar - 3.2.0-SNAPSHOT - - 4.0.0 - - pulsar-package-filesystem-storage - Apache Pulsar :: Package Management :: Filesystem Storage - - - - ${project.groupId} - pulsar-package-core - ${project.parent.version} - - - - com.google.guava - guava - - - - ${project.groupId} - testmocks - ${project.parent.version} - test - - - + + + pulsar-package-management + io.streamnative + 3.2.0-SNAPSHOT + + 4.0.0 + pulsar-package-filesystem-storage + Apache Pulsar :: Package Management :: Filesystem Storage + + + ${project.groupId} + pulsar-package-core + ${project.parent.version} + + + com.google.guava + guava + + + ${project.groupId} + testmocks + ${project.parent.version} + test + + diff --git a/pulsar-package-management/pom.xml b/pulsar-package-management/pom.xml index 750b9756134f2..b453be1d3dbea 100644 --- a/pulsar-package-management/pom.xml +++ b/pulsar-package-management/pom.xml @@ -1,4 +1,4 @@ - + - - - pulsar - org.apache.pulsar - 3.2.0-SNAPSHOT - .. - - 4.0.0 - - pulsar-package-management - pom - Apache Pulsar :: Package Management - - core - bookkeeper-storage - filesystem-storage - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - - spotbugs - verify - - check - - - - - - + + + pulsar + io.streamnative + 3.2.0-SNAPSHOT + .. + + 4.0.0 + pulsar-package-management + pom + Apache Pulsar :: Package Management + + core + bookkeeper-storage + filesystem-storage + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + + spotbugs + verify + + check + + + + + + diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index e132ef14ab0cc..eb89ec99e5e54 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT - pulsar-proxy Pulsar Proxy - ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} pulsar-broker-common ${project.version} - ${project.groupId} pulsar-common ${project.version} - ${project.groupId} pulsar-docs-tools ${project.version} - ${project.groupId} pulsar-websocket ${project.version} - org.apache.commons commons-lang3 - io.swagger swagger-annotations provided - org.eclipse.jetty jetty-server - org.eclipse.jetty jetty-alpn-conscrypt-server - org.eclipse.jetty jetty-servlet - org.eclipse.jetty jetty-servlets - org.eclipse.jetty jetty-proxy - org.glassfish.jersey.core jersey-server - org.glassfish.jersey.containers jersey-container-servlet-core - org.glassfish.jersey.media jersey-media-json-jackson - com.google.guava guava - com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - org.glassfish.jersey.inject jersey-hk2 - javax.xml.bind jaxb-api - com.sun.activation javax.activation - io.prometheus simpleclient - io.prometheus simpleclient_hotspot - io.prometheus simpleclient_servlet - io.prometheus simpleclient_jetty - ${project.groupId} pulsar-broker ${project.version} test - ${project.groupId} testmocks ${project.version} test - org.awaitility awaitility test - com.beust jcommander - org.apache.logging.log4j log4j-core - com.github.seancfoley ipaddress @@ -295,5 +264,4 @@ - diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/extensions/ProxyExtensions.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/extensions/ProxyExtensions.java index 75059fc0d2551..95a3bf032fe21 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/extensions/ProxyExtensions.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/extensions/ProxyExtensions.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketAddress; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -49,6 +50,9 @@ public class ProxyExtensions implements AutoCloseable { * @return the collection of extensions */ public static ProxyExtensions load(ProxyConfiguration conf) throws IOException { + if (conf.getProxyExtensions().isEmpty()) { + return new ProxyExtensions(Collections.emptyMap()); + } ExtensionsDefinitions definitions = ProxyExtensionsUtils.searchForExtensions( conf.getProxyExtensionsDirectory(), conf.getNarExtractionDirectory()); diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java index 7178a0ceda4db..db2969e3c3920 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java @@ -173,35 +173,43 @@ public class ProxyConfiguration implements PulsarConfiguration { @FieldContext( category = CATEGORY_BROKER_DISCOVERY, - doc = "The service url points to the broker cluster. URL must have the pulsar:// prefix." + doc = "If does not set metadataStoreUrl or configurationMetadataStoreUrl, this url should point to the" + + " discovery service provider." + + " URL must have the pulsar:// prefix. And does not support multi url yet." ) private String brokerServiceURL; @FieldContext( category = CATEGORY_BROKER_DISCOVERY, - doc = "The tls service url points to the broker cluster. URL must have the pulsar+ssl:// prefix." + doc = "If does not set metadataStoreUrl or configurationMetadataStoreUrl, this url should point to the" + + " discovery service provider." + + " URL must have the pulsar+ssl:// prefix. And does not support multi url yet." ) private String brokerServiceURLTLS; @FieldContext( category = CATEGORY_BROKER_DISCOVERY, - doc = "The web service url points to the broker cluster" + doc = "The web service url points to the discovery service provider of the broker cluster, and does not support" + + " multi url yet." ) private String brokerWebServiceURL; @FieldContext( category = CATEGORY_BROKER_DISCOVERY, - doc = "The tls web service url points to the broker cluster" + doc = "The tls web service url points to the discovery service provider of the broker cluster, and does not" + + " support multi url yet." ) private String brokerWebServiceURLTLS; @FieldContext( category = CATEGORY_BROKER_DISCOVERY, - doc = "The web service url points to the function worker cluster." + doc = "The web service url points to the discovery service provider of the function worker cluster, and does" + + " not support multi url yet." + " Only configure it when you setup function workers in a separate cluster" ) private String functionWorkerWebServiceURL; @FieldContext( category = CATEGORY_BROKER_DISCOVERY, - doc = "The tls web service url points to the function worker cluster." + doc = "The tls web service url points to the discovery service provider of the function worker cluster, and" + + " does not support multi url yet." + " Only configure it when you setup function workers in a separate cluster" ) private String functionWorkerWebServiceURLTLS; diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java index aa80b03613bee..1a98601f2a95d 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java @@ -162,11 +162,28 @@ public ProxyServiceStarter(String[] args) throws Exception { if (isNotBlank(config.getBrokerServiceURL())) { checkArgument(config.getBrokerServiceURL().startsWith("pulsar://"), "brokerServiceURL must start with pulsar://"); + ensureUrlNotContainsComma("brokerServiceURL", config.getBrokerServiceURL()); } - if (isNotBlank(config.getBrokerServiceURLTLS())) { checkArgument(config.getBrokerServiceURLTLS().startsWith("pulsar+ssl://"), "brokerServiceURLTLS must start with pulsar+ssl://"); + ensureUrlNotContainsComma("brokerServiceURLTLS", config.getBrokerServiceURLTLS()); + } + + if (isNotBlank(config.getBrokerWebServiceURL())) { + ensureUrlNotContainsComma("brokerWebServiceURL", config.getBrokerWebServiceURL()); + } + if (isNotBlank(config.getBrokerWebServiceURLTLS())) { + ensureUrlNotContainsComma("brokerWebServiceURLTLS", config.getBrokerWebServiceURLTLS()); + } + + if (isNotBlank(config.getFunctionWorkerWebServiceURL())) { + ensureUrlNotContainsComma("functionWorkerWebServiceURLTLS", + config.getFunctionWorkerWebServiceURL()); + } + if (isNotBlank(config.getFunctionWorkerWebServiceURLTLS())) { + ensureUrlNotContainsComma("functionWorkerWebServiceURLTLS", + config.getFunctionWorkerWebServiceURLTLS()); } if ((isBlank(config.getBrokerServiceURL()) && isBlank(config.getBrokerServiceURLTLS())) @@ -187,6 +204,11 @@ public ProxyServiceStarter(String[] args) throws Exception { } } + private void ensureUrlNotContainsComma(String paramName, String paramValue) { + checkArgument(!paramValue.contains(","), paramName + " does not support multi urls yet," + + " it should point to the discovery service provider."); + } + public static void main(String[] args) throws Exception { ProxyServiceStarter serviceStarter = new ProxyServiceStarter(args); try { diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConfigurationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConfigurationTest.java index 97a73c20b60d0..a9a562e04c899 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConfigurationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConfigurationTest.java @@ -20,6 +20,8 @@ import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.testng.annotations.Test; import java.beans.Introspector; @@ -36,6 +38,8 @@ import java.util.Properties; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; @Test(groups = "broker") public class ProxyConfigurationTest { @@ -134,4 +138,119 @@ public void testConvert() throws IOException { } } + @Test + public void testBrokerUrlCheck() throws IOException { + ProxyConfiguration configuration = new ProxyConfiguration(); + // brokerServiceURL must start with pulsar:// + configuration.setBrokerServiceURL("127.0.0.1:6650"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("brokerServiceURL must start with pulsar://"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("brokerServiceURL must start with pulsar://")); + } + } + configuration.setBrokerServiceURL("pulsar://127.0.0.1:6650"); + + // brokerServiceURLTLS must start with pulsar+ssl:// + configuration.setBrokerServiceURLTLS("pulsar://127.0.0.1:6650"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("brokerServiceURLTLS must start with pulsar+ssl://"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("brokerServiceURLTLS must start with pulsar+ssl://")); + } + } + + // brokerServiceURL did not support multi urls yet. + configuration.setBrokerServiceURL("pulsar://127.0.0.1:6650,pulsar://127.0.0.2:6650"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("brokerServiceURL does not support multi urls yet"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("does not support multi urls yet")); + } + } + configuration.setBrokerServiceURL("pulsar://127.0.0.1:6650"); + + // brokerServiceURLTLS did not support multi urls yet. + configuration.setBrokerServiceURLTLS("pulsar+ssl://127.0.0.1:6650,pulsar+ssl:127.0.0.2:6650"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("brokerServiceURLTLS does not support multi urls yet"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("does not support multi urls yet")); + } + } + configuration.setBrokerServiceURLTLS("pulsar+ssl://127.0.0.1:6650"); + + // brokerWebServiceURL did not support multi urls yet. + configuration.setBrokerWebServiceURL("http://127.0.0.1:8080,http://127.0.0.2:8080"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("brokerWebServiceURL does not support multi urls yet"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("does not support multi urls yet")); + } + } + configuration.setBrokerWebServiceURL("http://127.0.0.1:8080"); + + // brokerWebServiceURLTLS did not support multi urls yet. + configuration.setBrokerWebServiceURLTLS("https://127.0.0.1:443,https://127.0.0.2:443"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("brokerWebServiceURLTLS does not support multi urls yet"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("does not support multi urls yet")); + } + } + configuration.setBrokerWebServiceURLTLS("https://127.0.0.1:443"); + + // functionWorkerWebServiceURL did not support multi urls yet. + configuration.setFunctionWorkerWebServiceURL("http://127.0.0.1:8080,http://127.0.0.2:8080"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("functionWorkerWebServiceURL does not support multi urls yet"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("does not support multi urls yet")); + } + } + configuration.setFunctionWorkerWebServiceURL("http://127.0.0.1:8080"); + + // functionWorkerWebServiceURLTLS did not support multi urls yet. + configuration.setFunctionWorkerWebServiceURLTLS("http://127.0.0.1:443,http://127.0.0.2:443"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("functionWorkerWebServiceURLTLS does not support multi urls yet"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("does not support multi urls yet")); + } + } + configuration.setFunctionWorkerWebServiceURLTLS("http://127.0.0.1:443"); + } + } \ No newline at end of file diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java index 925e8192e145a..71b1087ee64b2 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java @@ -45,7 +45,7 @@ public class ProxyServiceStarterTest extends MockedPulsarServiceBaseTest { - static final String[] ARGS = new String[]{"-c", "./src/test/resources/proxy.conf"}; + public static final String[] ARGS = new String[]{"-c", "./src/test/resources/proxy.conf"}; protected ProxyServiceStarter serviceStarter; protected String serviceUrl; diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithExtensibleLoadManagerTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithExtensibleLoadManagerTest.java index 3a787a8b35995..f997532b2734c 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithExtensibleLoadManagerTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithExtensibleLoadManagerTest.java @@ -47,6 +47,7 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationService; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; @@ -97,6 +98,7 @@ private ServiceConfiguration configureExtensibleLoadManager(ServiceConfiguration config.setLoadBalancerInFlightServiceUnitStateWaitingTimeInMillis(5 * 1000); config.setLoadBalancerServiceUnitStateMonitorIntervalInSeconds(1); config.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); + config.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); config.setLoadBalancerSheddingEnabled(false); return config; } @@ -140,7 +142,7 @@ private String getDstBrokerLookupUrl(TopicName topicName) throws Exception { var srcBrokerUrl = admin.lookups().lookupTopic(topicName.toString()); return getAllBrokers().stream(). filter(pulsarService -> !Objects.equals(srcBrokerUrl, pulsarService.getBrokerServiceUrl())). - map(PulsarService::getLookupServiceAddress). + map(PulsarService::getBrokerId). findAny().orElseThrow(() -> new Exception("Could not determine destination broker lookup URL")); } diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml index a13b2db33104b..7f37240851bea 100644 --- a/pulsar-testclient/pom.xml +++ b/pulsar-testclient/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar - pulsar - 3.2.0-SNAPSHOT - .. - - - pulsar-testclient - Pulsar Test Client - Pulsar Test Client - - - - ${project.groupId} - testmocks - ${project.version} - test - - - - org.apache.zookeeper - zookeeper - - - ch.qos.logback - logback-core - - - ch.qos.logback - logback-classic - - - io.netty - netty-tcnative - - - - - - ${project.groupId} - pulsar-client-admin-original - ${project.version} - - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - - ${project.groupId} - pulsar-client-messagecrypto-bc - ${project.version} - true - - - - ${project.groupId} - pulsar-broker - ${project.version} - - - - ${project.groupId} - pulsar-cli-utils - ${project.version} - - - - commons-configuration - commons-configuration - - - - com.beust - jcommander - compile - - - - org.hdrhistogram - HdrHistogram - - - - com.fasterxml.jackson.core - jackson-databind - - - - org.awaitility - awaitility - test - - - - com.github.tomakehurst - wiremock-jre8 - ${wiremock.version} - test - - - - - - - - - test-jar-dependencies - - - maven.test.skip - !true - - - - - ${project.groupId} - pulsar-broker - ${project.version} - test-jar - test - - - - - - - - - org.gaul - modernizer-maven-plugin - - true - 8 - - - - modernizer - verify - - modernizer - - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - + + 4.0.0 + + io.streamnative + pulsar + 3.2.0-SNAPSHOT + .. + + pulsar-testclient + Pulsar Test Client + Pulsar Test Client + + + ${project.groupId} + testmocks + ${project.version} + test + + + org.apache.zookeeper + zookeeper + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + io.netty + netty-tcnative + + + + + ${project.groupId} + pulsar-client-admin-original + ${project.version} + + + ${project.groupId} + pulsar-client-original + ${project.version} + + + ${project.groupId} + pulsar-client-messagecrypto-bc + ${project.version} + true + + + ${project.groupId} + pulsar-broker + ${project.version} + + + ${project.groupId} + pulsar-cli-utils + ${project.version} + + + commons-configuration + commons-configuration + + + com.beust + jcommander + compile + + + org.hdrhistogram + HdrHistogram + + + com.fasterxml.jackson.core + jackson-databind + + + org.awaitility + awaitility + test + + + com.github.tomakehurst + wiremock-jre8 + ${wiremock.version} + test + + + + + + test-jar-dependencies + + + maven.test.skip + !true + + + + + ${project.groupId} + pulsar-broker + ${project.version} + test-jar + test + + + + + + + + org.gaul + modernizer-maven-plugin + + true + 8 + + + + modernizer + verify + + modernizer + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + diff --git a/pulsar-transaction/common/pom.xml b/pulsar-transaction/common/pom.xml index 627fd1a711511..30827f131bb0a 100644 --- a/pulsar-transaction/common/pom.xml +++ b/pulsar-transaction/common/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar - pulsar-transaction-parent - 3.2.0-SNAPSHOT - - - pulsar-transaction-common - Pulsar Transaction :: Common - - + + 4.0.0 + + io.streamnative + pulsar-transaction-parent + 3.2.0-SNAPSHOT + + pulsar-transaction-common + Pulsar Transaction :: Common + - - - - org.gaul - modernizer-maven-plugin - - true - 8 - - - - modernizer - verify - - modernizer - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - + + + + org.gaul + modernizer-maven-plugin + + true + 8 + + + + modernizer + verify + + modernizer + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + diff --git a/pulsar-transaction/common/src/main/resources/findbugsExclude.xml b/pulsar-transaction/common/src/main/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-transaction/common/src/main/resources/findbugsExclude.xml +++ b/pulsar-transaction/common/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - - org.apache.pulsar - pulsar-transaction-parent - 3.2.0-SNAPSHOT - - - pulsar-transaction-coordinator - Pulsar Transaction :: Coordinator - - - - - ${project.groupId} - pulsar-common - ${project.version} - - - - ${project.groupId} - managed-ledger - ${project.version} - - - - ${project.groupId} - testmocks - ${project.version} - test - - - - org.awaitility - awaitility - test - - - - - - - - org.gaul - modernizer-maven-plugin - - true - 8 - - - - modernizer - verify - - modernizer - - - - - - org.codehaus.mojo - properties-maven-plugin - - - initialize - - set-system-properties - - - - - proto_path - ${project.parent.parent.basedir} - - - proto_search_strategy - 2 - - - - - - - - com.github.splunk.lightproto - lightproto-maven-plugin - ${lightproto-maven-plugin.version} - - - - generate - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - + + 4.0.0 + + io.streamnative + pulsar-transaction-parent + 3.2.0-SNAPSHOT + + pulsar-transaction-coordinator + Pulsar Transaction :: Coordinator + + + ${project.groupId} + pulsar-common + ${project.version} + + + ${project.groupId} + managed-ledger + ${project.version} + + + ${project.groupId} + testmocks + ${project.version} + test + + + org.awaitility + awaitility + test + + + + + + org.gaul + modernizer-maven-plugin + + true + 8 + + + + modernizer + verify + + modernizer + + + + + + org.codehaus.mojo + properties-maven-plugin + + + initialize + + set-system-properties + + + + + proto_path + ${project.parent.parent.basedir} + + + proto_search_strategy + 2 + + + + + + + + com.github.splunk.lightproto + lightproto-maven-plugin + ${lightproto-maven-plugin.version} + + + + generate + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + diff --git a/pulsar-transaction/coordinator/src/main/resources/findbugsExclude.xml b/pulsar-transaction/coordinator/src/main/resources/findbugsExclude.xml index a81fce11f4d21..f1cad02b28a88 100644 --- a/pulsar-transaction/coordinator/src/main/resources/findbugsExclude.xml +++ b/pulsar-transaction/coordinator/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT - pulsar-transaction-parent Pulsar Transaction :: Parent - common coordinator - - - com.github.spotbugs spotbugs-maven-plugin @@ -68,7 +63,5 @@ - - diff --git a/pulsar-websocket/pom.xml b/pulsar-websocket/pom.xml index 900c5181bcaab..5cfef7b3498dd 100644 --- a/pulsar-websocket/pom.xml +++ b/pulsar-websocket/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - pulsar-websocket Pulsar WebSocket - ${project.groupId} pulsar-broker-common ${project.version} - ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} managed-ledger ${project.parent.version} test - ${project.groupId} pulsar-docs-tools @@ -63,57 +57,48 @@ - org.apache.commons commons-lang3 - org.glassfish.jersey.containers jersey-container-servlet-core - org.glassfish.jersey.containers jersey-container-servlet - org.glassfish.jersey.inject jersey-hk2 - com.google.code.gson gson - io.swagger swagger-core provided - com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - - + org.eclipse.jetty.websocket websocket-api ${jetty.version} - - + org.eclipse.jetty.websocket websocket-server ${jetty.version} - + org.eclipse.jetty.websocket javax-websocket-client-impl @@ -128,9 +113,7 @@ org.hdrhistogram HdrHistogram - - @@ -150,7 +133,6 @@ - com.github.spotbugs spotbugs-maven-plugin @@ -168,7 +150,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-websocket/src/main/resources/findbugsExclude.xml b/pulsar-websocket/src/main/resources/findbugsExclude.xml index c2b0d7dac0d3b..3686224162129 100644 --- a/pulsar-websocket/src/main/resources/findbugsExclude.xml +++ b/pulsar-websocket/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + @@ -209,4 +209,4 @@ - \ No newline at end of file + diff --git a/src/check-binary-license.sh b/src/check-binary-license.sh index 6aec8b7cf1bd9..c8c3750cf2ed7 100755 --- a/src/check-binary-license.sh +++ b/src/check-binary-license.sh @@ -53,7 +53,7 @@ EXIT=0 # Check all bundled jars are mentioned in LICENSE for J in $JARS; do - echo $J | grep -q "org.apache.pulsar" + echo $J | grep -q "io.streamnative" if [ $? == 0 ]; then continue fi diff --git a/src/findbugs-exclude.xml b/src/findbugs-exclude.xml index 8f289b83a7be1..de4d3f7a93371 100644 --- a/src/findbugs-exclude.xml +++ b/src/findbugs-exclude.xml @@ -1,4 +1,4 @@ - + - - - - - - - - - - - - - + + + + + + + + + + + + diff --git a/src/gen-pulsar-bom.sh b/src/gen-pulsar-bom.sh new file mode 100755 index 0000000000000..e9fb455d5ad3d --- /dev/null +++ b/src/gen-pulsar-bom.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Script to generate a BOM using the following approach: +# +# - do a local publish to get list of modules for BOM +# - for each published module add dependency entry +# - replace the current section with +# entries gathered in previous step + +set -e + +# Determine top level project directory +SRC_DIR=$(dirname "$0") +ROOT_DIR=`cd ${SRC_DIR}/..; pwd` +LOCAL_DEPLOY_DIR=${ROOT_DIR}/target/staging-deploy +PULSAR_BOM_DIR=${ROOT_DIR}/pulsar-bom + +pushd ${ROOT_DIR} > /dev/null +echo "Performing local publish to determine modules for BOM." +rm -rf ${LOCAL_DEPLOY_DIR} +./mvnw deploy -DaltDeploymentRepository=local::default::file:${LOCAL_DEPLOY_DIR} -DskipTests +./mvnw deploy -DaltDeploymentRepository=local::default::file:${LOCAL_DEPLOY_DIR} -DskipTests -f tests/pom.xml -pl org.apache.pulsar.tests:tests-parent,org.apache.pulsar.tests:integration +echo "$(ls ${LOCAL_DEPLOY_DIR}/org/apache/pulsar | wc -l) modules locally published to ${LOCAL_DEPLOY_DIR}." +popd > /dev/null + +DEPENDENCY_MGMT_PRE=$(cat <<-END + + +END +) +DEPENDENCY_BLOCK=$(cat <<-END + + org.apache.pulsar + @ARTIFACT_ID@ + \${project.version} + +END +) +DEPENDENCY_MGMT_POST=$(cat <<-END + + + +END +) +ALL_DEPS="" +NEWLINE=$'\n' + +pushd ${LOCAL_DEPLOY_DIR}/org/apache/pulsar/ > /dev/null +echo "Traversing locally published modules." +for f in */ +do + ARTIFACT_ID="${f%/}" + DEPENDENCY=$(echo "${DEPENDENCY_BLOCK/@ARTIFACT_ID@/$ARTIFACT_ID}") + if [ "${ARTIFACT_ID}" = "pulsar-bom" ]; then + continue + elif [ -z "$ALL_DEPS" ]; then + ALL_DEPS="$DEPENDENCY" + else + ALL_DEPS="$ALL_DEPS$NEWLINE$DEPENDENCY" + fi +done +popd > /dev/null + +POM_XML=$(<${PULSAR_BOM_DIR}/pom.xml) +POM_XML=$(echo "${POM_XML%%*}") +echo "$POM_XML$DEPENDENCY_MGMT_PRE$NEWLINE$ALL_DEPS$NEWLINE$DEPENDENCY_MGMT_POST" > ${PULSAR_BOM_DIR}/pom.xml + +echo "Created BOM ${PULSAR_BOM_DIR}/pom.xml." +echo "You must manually inspect changes and submit a PR with the changes." diff --git a/src/idea-code-style.xml b/src/idea-code-style.xml index 810dee34a3494..b49c523c322b2 100644 --- a/src/idea-code-style.xml +++ b/src/idea-code-style.xml @@ -1,3 +1,4 @@ + - - - - - \ No newline at end of file + diff --git a/src/owasp-dependency-check-false-positives.xml b/src/owasp-dependency-check-false-positives.xml index 345be8f4d2c06..151752c98f7a1 100644 --- a/src/owasp-dependency-check-false-positives.xml +++ b/src/owasp-dependency-check-false-positives.xml @@ -1,4 +1,4 @@ - + - - + + file name: zookeeper-3.8.0.jar - ]]> + e395c1d8a71557b7569cc6a83487b2e30e2e58fe CVE-2021-28164 - file name: zookeeper-3.8.0.jar - ]]> + e395c1d8a71557b7569cc6a83487b2e30e2e58fe CVE-2021-29425 - file name: zookeeper-3.8.0.jar - ]]> + e395c1d8a71557b7569cc6a83487b2e30e2e58fe CVE-2021-34429 - file name: zookeeper-prometheus-metrics-3.8.0.jar - ]]> + 849e8ece2845cb0185d721233906d487a7f1e4cf CVE-2021-28164 - file name: zookeeper-prometheus-metrics-3.8.0.jar - ]]> + 849e8ece2845cb0185d721233906d487a7f1e4cf CVE-2021-29425 - file name: zookeeper-prometheus-metrics-3.8.0.jar - ]]> + 849e8ece2845cb0185d721233906d487a7f1e4cf CVE-2021-34429 - file name: zookeeper-jute-3.8.0.jar - ]]> + 6560f966bcf1aa78d27bcfa75fb6c4463a72c6c5 CVE-2021-28164 - file name: zookeeper-jute-3.8.0.jar - ]]> + 6560f966bcf1aa78d27bcfa75fb6c4463a72c6c5 CVE-2021-29425 - file name: zookeeper-jute-3.8.0.jar - ]]> + 6560f966bcf1aa78d27bcfa75fb6c4463a72c6c5 CVE-2021-34429 - - file name: debezium-connector-postgres-1.7.2.Final.jar - ]]> + 69c1edfa7d89531af511fcd07e8516fa450f746a CVE-2021-23214 - - - + - file name: mariadb-java-client-2.7.5.jar - ]]> + 9dd29797ecabe7d2e7fa892ec6713a5552cfcc59 CVE-2022-27376 CVE-2022-27377 @@ -162,43 +159,36 @@ CVE-2022-27386 CVE-2022-27387 - - file name: google-http-client-gson-1.41.0.jar - ]]> + 1a754a5dd672218a2ac667d7ff2b28df7a5a240e CVE-2022-25647 - commons-net is not used at all and therefore commons-net vulnerability CVE-2021-37533 is a false positive. CVE-2021-37533 - fredsmith utils library is not used at all. CVE-2021-4277 is a false positive. CVE-2021-4277 - It treat pulsar-io-kafka-connect-adaptor as a lib of Kafka, CVE-2021-25194 is a false positive. CVE-2023-25194 - It treat pulsar-io-kafka-connect-adaptor as a lib of Kafka, CVE-2021-34917 is a false positive. CVE-2022-34917 - yaml_project is not used at all. Any CVEs reported for yaml_project are false positives. cpe:/a:yaml_project:yaml - flat_project is not used at all. cpe:/a:flat_project:flat - \ No newline at end of file + diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml index 1ce7392a4898d..8d3af892c3385 100644 --- a/src/owasp-dependency-check-suppressions.xml +++ b/src/owasp-dependency-check-suppressions.xml @@ -1,4 +1,4 @@ - + - - - Ignore netty CVEs in GRPC shaded Netty. - .*grpc-netty-shaded.* - cpe:/a:netty:netty - - - Suppress libthrift-0.12.0.jar vulnerabilities - org.apache.thrift:libthrift:0.12.0 - .* - - - - - + + Ignore netty CVEs in GRPC shaded Netty. + .*grpc-netty-shaded.* + cpe:/a:netty:netty + + + Suppress libthrift-0.12.0.jar vulnerabilities + org.apache.thrift:libthrift:0.12.0 + .* + + + + file name: msgpack-core-0.9.0.jar - ]]> - 87d9ce0b22de48428fa32bb8ad476e18b6969548 - CVE-2022-41719 - - - - - + 87d9ce0b22de48428fa32bb8ad476e18b6969548 + CVE-2022-41719 + + + + file name: elasticsearch-java-8.1.0.jar CVE-2022-23712 is only related to Elastic server. - ]]> - edf5be04cbc2eafc51540ba33f9536e788b9d60b - CVE-2022-23712 - - - + edf5be04cbc2eafc51540ba33f9536e788b9d60b + CVE-2022-23712 + + + file name: elasticsearch-rest-client-8.1.0.jar CVE-2022-23712 is only related to Elastic server. - ]]> - 10e7aa09f10955a074c0a574cb699344d2745df1 - CVE-2022-23712 - - - - - + 10e7aa09f10955a074c0a574cb699344d2745df1 + CVE-2022-23712 + + + + file name: kotlin-stdlib-common-1.4.32.jar - ]]> - ef50bfa2c0491a11dcc35d9822edbfd6170e1ea2 - cpe:/a:jetbrains:kotlin - - - + ef50bfa2c0491a11dcc35d9822edbfd6170e1ea2 + cpe:/a:jetbrains:kotlin + + + file name: kotlin-stdlib-jdk7-1.4.32.jar - ]]> - 3546900a3ebff0c43f31190baf87a9220e37b7ea - CVE-2022-24329 - - - + 3546900a3ebff0c43f31190baf87a9220e37b7ea + CVE-2022-24329 + + + file name: kotlin-stdlib-jdk8-1.4.32.jar - ]]> - 3302f9ec8a5c1ed220781dbd37770072549bd333 - CVE-2022-24329 - - - + 3302f9ec8a5c1ed220781dbd37770072549bd333 + CVE-2022-24329 + + + file name: kotlin-stdlib-1.4.32.jar - ]]> - 461367948840adbb0839c51d91ed74ef4a9ccb52 - CVE-2022-24329 - - - - - + 461367948840adbb0839c51d91ed74ef4a9ccb52 + CVE-2022-24329 + + + + file name: canal.client-1.1.5.jar (shaded: com.google.guava:guava:22.0) - ]]> - b87878db57d5cfc2ca7d3972cc8f7486bf02fbca - CVE-2018-10237 - - - + b87878db57d5cfc2ca7d3972cc8f7486bf02fbca + CVE-2018-10237 + + + file name: canal.client-1.1.5.jar (shaded: com.google.guava:guava:22.0) - ]]> - b87878db57d5cfc2ca7d3972cc8f7486bf02fbca - CVE-2020-8908 - - - + b87878db57d5cfc2ca7d3972cc8f7486bf02fbca + CVE-2020-8908 + + + file name: canal.client-1.1.5.jar (shaded: com.google.guava:guava:32.1.1) CVE cannot take effect. Already covered by PR https://github.com/apache/pulsar/pull/20699 - ]]> - b87878db57d5cfc2ca7d3972cc8f7486bf02fbca - CVE-2023-2976 - - - - + b87878db57d5cfc2ca7d3972cc8f7486bf02fbca + CVE-2023-2976 + + + + file name: avro-1.10.2.jar - ]]> - a65aaa91c1aeceb3dd4859dbb9765d1c2063f5a2 - CVE-2021-43045 - - - + a65aaa91c1aeceb3dd4859dbb9765d1c2063f5a2 + CVE-2021-43045 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2018-14668 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2018-14668 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2018-14669 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2018-14669 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2018-14670 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2018-14670 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2018-14671 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2018-14671 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2018-14672 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2018-14672 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2019-15024 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2019-15024 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2019-16535 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2019-16535 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2019-18657 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2019-18657 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2021-25263 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2021-25263 + + + file name: clickhouse-jdbc-0.4.6-all.jar (shaded: com.google.guava:guava:32.1.1) CVE cannot take effect. Already covered by PR https://github.com/apache/pulsar/pull/20699 - ]]> - d3b929509399a698915b24ff47db781d0c526760 - CVE-2023-2976 - - - + d3b929509399a698915b24ff47db781d0c526760 + CVE-2023-2976 + + + file name: logback-core-1.1.3.jar - ]]> - e3c02049f2dbbc764681b40094ecf0dcbc99b157 - cpe:/a:qos:logback - - - + e3c02049f2dbbc764681b40094ecf0dcbc99b157 + cpe:/a:qos:logback + + + file name: rocketmq-acl-4.5.2.jar - ]]> - 0e2bd9c162280cd79c2ea0f67f174ee5d7b84ddd - cpe:/a:apache:rocketmq - - - - ^pkg:maven/org\.springframework/spring.*$ - CVE-2016-1000027 - - - + 0e2bd9c162280cd79c2ea0f67f174ee5d7b84ddd + cpe:/a:apache:rocketmq + + + Ignored since we are not vulnerable + ^pkg:maven/org\.springframework/spring.*$ + CVE-2016-1000027 + + + file name: logback-classic-1.1.3.jar - ]]> - d90276fff414f06cb375f2057f6778cd63c6082f - cpe:/a:qos:logback - - - + d90276fff414f06cb375f2057f6778cd63c6082f + cpe:/a:qos:logback + + + file name: logback-core-1.1.3.jar - ]]> - e3c02049f2dbbc764681b40094ecf0dcbc99b157 - CVE-2017-5929 - - - + e3c02049f2dbbc764681b40094ecf0dcbc99b157 + CVE-2017-5929 + + + file name: logback-classic-1.1.3.jar - ]]> - d90276fff414f06cb375f2057f6778cd63c6082f - CVE-2017-5929 - - - + d90276fff414f06cb375f2057f6778cd63c6082f + CVE-2017-5929 + + + file name: logback-classic-1.1.3.jar - ]]> - d90276fff414f06cb375f2057f6778cd63c6082f - CVE-2021-42550 - - - - Ignore etdc CVEs in jetcd - .*jetcd.* - cpe:/a:etcd:etcd - - - Ignore etdc CVEs in jetcd - .*jetcd.* - cpe:/a:redhat:etcd - - - Ignore grpc CVEs in jetcd - .*jetcd-grpc.* - cpe:/a:grpc:grpc - - - - - + d90276fff414f06cb375f2057f6778cd63c6082f + CVE-2021-42550 + + + Ignore etdc CVEs in jetcd + .*jetcd.* + cpe:/a:etcd:etcd + + + Ignore etdc CVEs in jetcd + .*jetcd.* + cpe:/a:redhat:etcd + + + Ignore grpc CVEs in jetcd + .*jetcd-grpc.* + cpe:/a:grpc:grpc + + + + file name: bc-fips-1.0.2.jar - ]]> - 4fb5db5f03d00f6a94e43b78d097978190e4abb2 - CVE-2020-26939 - - - + 4fb5db5f03d00f6a94e43b78d097978190e4abb2 + CVE-2020-26939 + + + file name: bcpkix-fips-1.0.2.jar - ]]> - 543bc7a08cdba0172e95e536b5f7ca61f021253d - CVE-2020-15522 - - - + 543bc7a08cdba0172e95e536b5f7ca61f021253d + CVE-2020-15522 + + + file name: bcpkix-fips-1.0.2.jar - ]]> - 543bc7a08cdba0172e95e536b5f7ca61f021253d - CVE-2020-26939 - - - - - + 543bc7a08cdba0172e95e536b5f7ca61f021253d + CVE-2020-26939 + + + + file name: openstack-swift-2.5.0.jar - ]]> - d99d0eab2e01d69d8a326fc152427fbd759af88a - CVE-2016-0738 - - - + d99d0eab2e01d69d8a326fc152427fbd759af88a + CVE-2016-0738 + + + file name: openstack-swift-2.5.0.jar - ]]> - d99d0eab2e01d69d8a326fc152427fbd759af88a - CVE-2017-16613 - - - + d99d0eab2e01d69d8a326fc152427fbd759af88a + CVE-2017-16613 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2018-14432 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2018-14432 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2018-20170 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2018-20170 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2020-12689 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2020-12689 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2020-12690 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2020-12690 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2020-12691 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2020-12691 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2020-12692 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2020-12692 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2021-3563 - - - - - + file name: org.apache.pulsar:pulsar-io-solr:2.10.0-SNAPSHOT - ]]> - ^pkg:maven/org\.apache\.pulsar/pulsar\-io\-solr@.*-SNAPSHOT$ - cpe:/a:apache:pulsar - - - + ^pkg:maven/org\.apache\.pulsar/pulsar\-io\-solr@.*-SNAPSHOT$ + cpe:/a:apache:pulsar + + + file name: org.apache.pulsar:pulsar-io-solr:2.10.0-SNAPSHOT - ]]> - ^pkg:maven/org\.apache\.pulsar/pulsar\-io\-solr@.*-SNAPSHOT$ - cpe:/a:apache:solr - - - - - + ^pkg:maven/org\.apache\.pulsar/pulsar\-io\-solr@.*-SNAPSHOT$ + cpe:/a:apache:solr + + + + file name: debezium-connector-mysql-1.9.7.Final.jar - ]]> - 74c623b4a9b231e2f0e8f6053b38abd3bc487ce2 - CVE-2017-15945 - - - + 74c623b4a9b231e2f0e8f6053b38abd3bc487ce2 + CVE-2017-15945 + + + file name: mysql-binlog-connector-java-0.27.2.jar - ]]> - 23294cd730e29c00b8ddfbde517dfc955aba090f - CVE-2017-15945 - - - + 23294cd730e29c00b8ddfbde517dfc955aba090f + CVE-2017-15945 + + + file name: debezium-connector-postgres-1.9.7.Final.jar - ]]> - 300ff0bbf795643e914b7c8a6d6ba812a8354d62 - CVE-2015-0241 - CVE-2015-0242 - CVE-2015-0243 - CVE-2015-0244 - CVE-2015-3166 - CVE-2015-3167 - CVE-2016-0766 - CVE-2016-0768 - CVE-2016-0773 - CVE-2016-5423 - CVE-2016-5424 - CVE-2016-7048 - CVE-2017-14798 - CVE-2017-7484 - CVE-2018-1115 - CVE-2019-10127 - CVE-2019-10128 - CVE-2019-10210 - CVE-2019-10211 - CVE-2020-25694 - CVE-2020-25695 - CVE-2021-23214 - - - + 300ff0bbf795643e914b7c8a6d6ba812a8354d62 + CVE-2015-0241 + CVE-2015-0242 + CVE-2015-0243 + CVE-2015-0244 + CVE-2015-3166 + CVE-2015-3167 + CVE-2016-0766 + CVE-2016-0768 + CVE-2016-0773 + CVE-2016-5423 + CVE-2016-5424 + CVE-2016-7048 + CVE-2017-14798 + CVE-2017-7484 + CVE-2018-1115 + CVE-2019-10127 + CVE-2019-10128 + CVE-2019-10210 + CVE-2019-10211 + CVE-2020-25694 + CVE-2020-25695 + CVE-2021-23214 + + + file name: protostream-types-4.4.1.Final.jar - ]]> - 29b45ebea1e4ce62ab3ec5eb76fa9771f98941b0 - CVE-2016-0750 - CVE-2017-15089 - CVE-2017-2638 - CVE-2019-10158 - CVE-2019-10174 - CVE-2020-25711 - - - + 29b45ebea1e4ce62ab3ec5eb76fa9771f98941b0 + CVE-2016-0750 + CVE-2017-15089 + CVE-2017-2638 + CVE-2019-10158 + CVE-2019-10174 + CVE-2020-25711 + + + file name: mariadb-java-client-2.7.5.jar - ]]> - 9dd29797ecabe7d2e7fa892ec6713a5552cfcc59 - CVE-2020-28912 - CVE-2021-46669 - CVE-2021-46666 - CVE-2021-46667 - - - - + 9dd29797ecabe7d2e7fa892ec6713a5552cfcc59 + CVE-2020-28912 + CVE-2021-46669 + CVE-2021-46666 + CVE-2021-46667 + + + + file name: cassandra-driver-core-3.11.2.jar - ]]> - e0aad9f8611e710b9a0ce49747f7465ce07d8404 - CVE-2020-17516 - CVE-2021-44521 - - - + e0aad9f8611e710b9a0ce49747f7465ce07d8404 + CVE-2020-17516 + CVE-2021-44521 + + + The vulnerable method is deprecated in Guava, but isn't removed. It's necessary to suppress this CVE. See https://github.com/google/guava/issues/4011 - ]]> - CVE-2020-8908 - - - + CVE-2020-8908 + + + This is a false positive in jackson-databind. See https://github.com/FasterXML/jackson-databind/issues/3972#issuecomment-1596604021 - ]]> - CVE-2023-35116 - - - + CVE-2023-35116 + + + This is a false positive in avro-protobuf. The vulnerability is in Hamba avro golang library. - ]]> - CVE-2023-37475 - - - + CVE-2023-37475 + + + This CVE can be suppressed since it is covered in Pulsar by hostname verification changes made in https://github.com/apache/pulsar/pull/15824. - ]]> - CVE-2023-4586 - + + CVE-2023-4586 + diff --git a/src/settings.xml b/src/settings.xml index 80ec2f40620e7..8fc2f13a37a59 100644 --- a/src/settings.xml +++ b/src/settings.xml @@ -1,3 +1,4 @@ + apache.releases.https @@ -34,7 +34,6 @@ ${env.APACHE_PASSWORD} - apache diff --git a/structured-event-log/pom.xml b/structured-event-log/pom.xml index 72e9273b7c325..56233522f9051 100644 --- a/structured-event-log/pom.xml +++ b/structured-event-log/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - structured-event-log Structured event logger - org.slf4j slf4j-api - org.apache.logging.log4j @@ -60,7 +56,6 @@ test - @@ -78,5 +73,4 @@ - diff --git a/testmocks/pom.xml b/testmocks/pom.xml index 6f362bc254d6c..99b1d400850c7 100644 --- a/testmocks/pom.xml +++ b/testmocks/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - pulsar - org.apache.pulsar + io.streamnative 3.2.0-SNAPSHOT - testmocks jar Pulsar Test Mocks - - org.apache.zookeeper zookeeper @@ -52,29 +48,23 @@ - org.apache.bookkeeper bookkeeper-server - org.apache.commons commons-lang3 - org.testng testng - org.objenesis objenesis - - @@ -92,5 +82,4 @@ - diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java index f0d279ef25050..998ef73fbd3e9 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java @@ -18,6 +18,7 @@ */ package org.apache.bookkeeper.client; +import static com.google.common.base.Preconditions.checkArgument; import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.Arrays; @@ -364,6 +365,7 @@ public synchronized CompletableFuture promiseAfter(int steps, List= 0, "The delay time must not be negative."); addEntryDelaysMillis.add(unit.toMillis(delay)); } diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java b/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java index 4d08a7f80df5b..cf08cc635106e 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java @@ -18,6 +18,7 @@ */ package org.apache.bookkeeper.client; +import static com.google.common.base.Preconditions.checkArgument; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; @@ -65,6 +66,7 @@ public void addCount(long delta) { @Override public void addLatency(long eventLatency, TimeUnit unit) { + checkArgument(eventLatency >= 0, "The event latency must not be negative."); long valueMillis = unit.toMillis(eventLatency); updateMax(val.addAndGet(valueMillis)); } diff --git a/tests/bc_2_0_0/pom.xml b/tests/bc_2_0_0/pom.xml index ab0f53c5c888b..a0f1e659a6d90 100644 --- a/tests/bc_2_0_0/pom.xml +++ b/tests/bc_2_0_0/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar.tests - tests-parent - 3.2.0-SNAPSHOT - - - bc_2_0_0 - jar - Apache Pulsar :: Tests :: Backwards Client Compatibility 2.0.0-rc1-incubating - - - - - org.apache.pulsar - pulsar-client - 2.0.0-rc1-incubating - test - - - - org.testcontainers - testcontainers - test - - - - - + + 4.0.0 + + io.streamnative.tests + tests-parent + 3.2.0-SNAPSHOT + + bc_2_0_0 + jar + Apache Pulsar :: Tests :: Backwards Client Compatibility 2.0.0-rc1-incubating + + + org.apache.pulsar + pulsar-client + 2.0.0-rc1-incubating + test + + + org.testcontainers + testcontainers + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + BackwardsCompatTests + + + BackwardsCompatTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - BackwardsCompatTests - - - BackwardsCompatTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/pulsar.xml - - 1 - - - - - - + false + + src/test/resources/pulsar.xml + + 1 + + + + + + diff --git a/tests/bc_2_0_0/src/test/resources/pulsar.xml b/tests/bc_2_0_0/src/test/resources/pulsar.xml index 43dfaea15813c..2e9d57ed72832 100644 --- a/tests/bc_2_0_0/src/test/resources/pulsar.xml +++ b/tests/bc_2_0_0/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/bc_2_0_1/pom.xml b/tests/bc_2_0_1/pom.xml index cb71e62118840..ef8d6a0563aac 100644 --- a/tests/bc_2_0_1/pom.xml +++ b/tests/bc_2_0_1/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar.tests - tests-parent - 3.2.0-SNAPSHOT - - - bc_2_0_1 - jar - Apache Pulsar :: Tests :: Backwards Client Compatibility 2.0.1-incubating - - - - - org.apache.pulsar - pulsar-client - 2.0.1-incubating - test - - - - org.testcontainers - testcontainers - test - - - - - + + 4.0.0 + + io.streamnative.tests + tests-parent + 3.2.0-SNAPSHOT + + bc_2_0_1 + jar + Apache Pulsar :: Tests :: Backwards Client Compatibility 2.0.1-incubating + + + org.apache.pulsar + pulsar-client + 2.0.1-incubating + test + + + org.testcontainers + testcontainers + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + BackwardsCompatTests + + + BackwardsCompatTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - BackwardsCompatTests - - - BackwardsCompatTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/pulsar.xml - - 1 - - - - - - + false + + src/test/resources/pulsar.xml + + 1 + + + + + + diff --git a/tests/bc_2_0_1/src/test/resources/pulsar.xml b/tests/bc_2_0_1/src/test/resources/pulsar.xml index 43dfaea15813c..2e9d57ed72832 100644 --- a/tests/bc_2_0_1/src/test/resources/pulsar.xml +++ b/tests/bc_2_0_1/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/bc_2_6_0/pom.xml b/tests/bc_2_6_0/pom.xml index ea51db62e3366..363a56eca8eb0 100644 --- a/tests/bc_2_6_0/pom.xml +++ b/tests/bc_2_6_0/pom.xml @@ -1,4 +1,4 @@ - + - - - org.apache.pulsar.tests - tests-parent - 3.2.0-SNAPSHOT - - 4.0.0 - - bc_2_6_0 - jar - Apache Pulsar :: Tests :: Backwards Client Compatibility 2.6.0 - - - - - com.google.code.gson - gson - test - - - - org.apache.pulsar - pulsar-client - 2.6.0 - test - - - - org.testcontainers - testcontainers - test - - - - - + + + io.streamnative.tests + tests-parent + 3.2.0-SNAPSHOT + + 4.0.0 + bc_2_6_0 + jar + Apache Pulsar :: Tests :: Backwards Client Compatibility 2.6.0 + + + com.google.code.gson + gson + test + + + org.apache.pulsar + pulsar-client + 2.6.0 + test + + + org.testcontainers + testcontainers + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + BackwardsCompatTests + + + BackwardsCompatTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - BackwardsCompatTests - - - BackwardsCompatTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/backwards-client.xml - - 1 - - - - - - - - + false + + src/test/resources/backwards-client.xml + + 1 + + + + + + diff --git a/tests/bc_2_6_0/src/test/resources/backwards-client.xml b/tests/bc_2_6_0/src/test/resources/backwards-client.xml index 43dfaea15813c..2e9d57ed72832 100644 --- a/tests/bc_2_6_0/src/test/resources/backwards-client.xml +++ b/tests/bc_2_6_0/src/test/resources/backwards-client.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/docker-images/java-test-functions/pom.xml b/tests/docker-images/java-test-functions/pom.xml index 1a8656e4b6d1f..81f45ac178456 100644 --- a/tests/docker-images/java-test-functions/pom.xml +++ b/tests/docker-images/java-test-functions/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar.tests + io.streamnative.tests docker-images 3.2.0-SNAPSHOT @@ -30,12 +30,12 @@ Apache Pulsar :: Tests :: Docker Images :: Java Test Functions - org.apache.pulsar + io.streamnative pulsar-io-core ${project.version} - org.apache.pulsar + io.streamnative pulsar-functions-api ${project.version} @@ -59,7 +59,6 @@ jar - docker @@ -68,15 +67,13 @@ integrationTests - - org.apache.pulsar + io.streamnative pulsar-functions-api-examples ${project.version} - @@ -91,7 +88,7 @@ - org.apache.pulsar:pulsar-functions-api-examples + io.streamnative:pulsar-functions-api-examples @@ -102,5 +99,4 @@ - diff --git a/tests/docker-images/java-test-image/pom.xml b/tests/docker-images/java-test-image/pom.xml index 67a7bd2eb02f6..9c621f2f1fb1b 100644 --- a/tests/docker-images/java-test-image/pom.xml +++ b/tests/docker-images/java-test-image/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar.tests + io.streamnative.tests docker-images 3.2.0-SNAPSHOT @@ -29,7 +29,6 @@ java-test-image Apache Pulsar :: Tests :: Docker Images :: Java Test Image pom - docker @@ -46,12 +45,12 @@ - org.apache.pulsar.tests + io.streamnative.tests java-test-functions ${project.parent.version} - org.apache.pulsar + io.streamnative pulsar-server-distribution ${project.parent.version} bin @@ -79,7 +78,7 @@ - org.apache.pulsar.tests + io.streamnative.tests java-test-functions ${project.parent.version} jar @@ -88,7 +87,7 @@ java-test-functions.jar - org.apache.pulsar + io.streamnative pulsar-server-distribution ${project.parent.version} bin diff --git a/tests/docker-images/java-test-plugins/pom.xml b/tests/docker-images/java-test-plugins/pom.xml index 1c472740220d1..2e44a12d6dcd9 100644 --- a/tests/docker-images/java-test-plugins/pom.xml +++ b/tests/docker-images/java-test-plugins/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar.tests + io.streamnative.tests docker-images 3.2.0-SNAPSHOT @@ -31,7 +31,7 @@ jar - org.apache.pulsar + io.streamnative pulsar-broker ${project.version} provided @@ -45,5 +45,4 @@ - diff --git a/tests/docker-images/latest-version-image/conf/functions_worker.conf b/tests/docker-images/latest-version-image/conf/functions_worker.conf index 8072639a0d4a2..6feb660231cec 100644 --- a/tests/docker-images/latest-version-image/conf/functions_worker.conf +++ b/tests/docker-images/latest-version-image/conf/functions_worker.conf @@ -22,7 +22,7 @@ autostart=false redirect_stderr=true stdout_logfile=/var/log/pulsar/functions_worker.log directory=/pulsar -environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC" +environment=PULSAR_MEM="-Xmx128M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/pulsar/logs/functions",PULSAR_GC="-XX:+UseZGC" command=/pulsar/bin/pulsar functions-worker user=pulsar stopwaitsecs=15 \ No newline at end of file diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml index e5d623e843bd4..682fe9fe2930d 100644 --- a/tests/docker-images/latest-version-image/pom.xml +++ b/tests/docker-images/latest-version-image/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar.tests + io.streamnative.tests docker-images 3.2.0-SNAPSHOT @@ -29,7 +29,6 @@ latest-version-image Apache Pulsar :: Tests :: Docker Images :: Latest Version Testing pom - docker @@ -40,17 +39,17 @@ - org.apache.pulsar.tests + io.streamnative.tests java-test-functions ${project.parent.version} - org.apache.pulsar.tests + io.streamnative.tests java-test-plugins ${project.parent.version} - org.apache.pulsar + io.streamnative pulsar-all-docker-image ${project.parent.version} pom @@ -77,7 +76,7 @@ - org.apache.pulsar.tests + io.streamnative.tests java-test-functions ${project.parent.version} jar @@ -86,7 +85,7 @@ java-test-functions.jar - org.apache.pulsar.tests + io.streamnative.tests java-test-plugins ${project.parent.version} jar diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml index 3e3d5edbd02e1..14ca807322753 100644 --- a/tests/docker-images/pom.xml +++ b/tests/docker-images/pom.xml @@ -1,4 +1,4 @@ - + - + pom 4.0.0 - org.apache.pulsar.tests + io.streamnative.tests tests-parent 3.2.0-SNAPSHOT diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index 3b07560403b4e..f48021ce6bcdd 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar.tests + io.streamnative.tests tests-parent 3.2.0-SNAPSHOT - integration jar Apache Pulsar :: Tests :: Integration - pulsar.xml 4.1.2 - com.google.code.gson @@ -44,37 +40,37 @@ test - org.apache.pulsar + io.streamnative pulsar-functions-api-examples ${project.version} test - org.apache.pulsar + io.streamnative pulsar-broker ${project.version} test - org.apache.pulsar + io.streamnative pulsar-common ${project.version} test - org.apache.pulsar + io.streamnative pulsar-client-original ${project.version} test - org.apache.pulsar + io.streamnative pulsar-client-admin-original ${project.version} test - org.apache.pulsar + io.streamnative managed-ledger ${project.version} test @@ -96,31 +92,27 @@ test - org.apache.pulsar + io.streamnative pulsar-io-kafka ${project.version} test - org.testcontainers mysql test - org.postgresql postgresql ${postgresql-jdbc.version} runtime - org.testcontainers postgresql test - com.github.docker-java docker-java-core @@ -131,65 +123,54 @@ bcpkix-jdk18on test - - org.apache.pulsar + io.streamnative pulsar-io-jdbc-postgres ${project.version} test - com.fasterxml.jackson.core jackson-databind test - com.fasterxml.jackson.dataformat jackson-dataformat-yaml test - org.opensearch.client opensearch-rest-high-level-client test - co.elastic.clients elasticsearch-java test - org.testcontainers elasticsearch test - - com.rabbitmq amqp-client ${rabbitmq-client.version} test - joda-time joda-time test - org.awaitility awaitility test - - + org.testcontainers localstack @@ -206,7 +187,6 @@ aws-java-sdk-core test - org.testcontainers @@ -219,9 +199,7 @@ ${mongo-reactivestreams.version} test - - @@ -262,7 +240,6 @@ - integrationTests diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarStateTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarStateTest.java index 8472ed3db2c2b..856e4edfea023 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarStateTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarStateTest.java @@ -25,13 +25,16 @@ import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; import static org.testng.Assert.fail; import com.google.common.base.Utf8; +import com.google.gson.Gson; import java.util.Base64; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Producer; @@ -95,10 +98,10 @@ private void doTestPythonWordCountFunction(String functionName) throws Exception getFunctionStatus(functionName, numMessages); // get state - queryState(functionName, "hello", numMessages); - queryState(functionName, "test", numMessages); + queryState(functionName, "hello", numMessages, numMessages - 1); + queryState(functionName, "test", numMessages, numMessages - 1); for (int i = 0; i < numMessages; i++) { - queryState(functionName, "message-" + i, 1); + queryState(functionName, "message-" + i, 1, 0); } // test put state @@ -110,6 +113,20 @@ private void doTestPythonWordCountFunction(String functionName) throws Exception String expectNumber = "\"numberValue\": 20"; putAndQueryState(functionName, "test-number", numberState, expectNumber); + byte[] valueBytes = Base64.getDecoder().decode(VALUE_BASE64); + String bytesString = Base64.getEncoder().encodeToString(valueBytes); + String byteState = "{\"key\":\"test-bytes\",\"byteValue\":\"" + bytesString + "\"}"; + putAndQueryStateByte(functionName, "test-bytes", byteState, valueBytes); + + String valueStr = "hello pulsar"; + byte[] valueStrBytes = valueStr.getBytes(UTF_8); + String bytesStrString = Base64.getEncoder().encodeToString(valueStrBytes); + String byteStrState = "{\"key\":\"test-str-bytes\",\"byteValue\":\"" + bytesStrString + "\"}"; + putAndQueryState(functionName, "test-str-bytes", byteStrState, valueStr); + + String byteStrStateWithEmptyValues = "{\"key\":\"test-str-bytes\",\"byteValue\":\"" + bytesStrString + "\",\"stringValue\":\"\",\"numberValue\":0}"; + putAndQueryState(functionName, "test-str-bytes", byteStrStateWithEmptyValues, valueStr); + // delete function deleteFunction(functionName); @@ -144,6 +161,22 @@ public void testSourceState() throws Exception { assertEquals(functionState.getStringValue(), "val1"); } + // query a non-exist key should get a 404 error + { + PulsarAdminException e = expectThrows(PulsarAdminException.class, () -> { + admin.functions().getFunctionState("public", "default", sourceName, "non-exist"); + }); + assertEquals(e.getStatusCode(), 404); + } + + // query a non-exist instance should get a 404 error + { + PulsarAdminException e = expectThrows(PulsarAdminException.class, () -> { + admin.functions().getFunctionState("public", "default", "non-exist", "non-exist"); + }); + assertEquals(e.getStatusCode(), 404); + } + Awaitility.await().ignoreExceptions().untilAsserted(() -> { FunctionState functionState = admin.functions().getFunctionState("public", "default", sourceName, "now"); assertTrue(functionState.getStringValue().matches("val1-.*")); @@ -186,6 +219,22 @@ public void testSinkState() throws Exception { assertEquals(functionState.getStringValue(), "val1"); } + // query a non-exist key should get a 404 error + { + PulsarAdminException e = expectThrows(PulsarAdminException.class, () -> { + admin.functions().getFunctionState("public", "default", sinkName, "non-exist"); + }); + assertEquals(e.getStatusCode(), 404); + } + + // query a non-exist instance should get a 404 error + { + PulsarAdminException e = expectThrows(PulsarAdminException.class, () -> { + admin.functions().getFunctionState("public", "default", "non-exist", "non-exist"); + }); + assertEquals(e.getStatusCode(), 404); + } + for (int i = 0; i < numMessages; i++) { producer.send("foo"); } @@ -208,6 +257,20 @@ public void testSinkState() throws Exception { getSinkInfoNotFound(sinkName); } + @Test(groups = {"python_state", "state", "function", "python_function"}) + public void testNonExistFunction() throws Exception { + String functionName = "non-exist-function-" + randomName(8); + try (PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(container.getHttpServiceUrl()).build()) { + // query a non-exist instance should get a 404 error + { + PulsarAdminException e = expectThrows(PulsarAdminException.class, () -> { + admin.functions().getFunctionState("public", "default", functionName, "non-exist"); + }); + assertEquals(e.getStatusCode(), 404); + } + } + } + @Test(groups = {"java_state", "state", "function", "java_function"}) public void testBytes2StringNotUTF8() { byte[] valueBytes = Base64.getDecoder().decode(VALUE_BASE64); @@ -450,7 +513,7 @@ private void getFunctionStatus(String functionName, int numMessages) throws Exce assertTrue(result.getStdout().contains("\"numSuccessfullyProcessed\" : " + numMessages)); } - private void queryState(String functionName, String key, int amount) + private void queryState(String functionName, String key, int amount, long version) throws Exception { ContainerExecResult result = container.execCmd( PulsarCluster.ADMIN_SCRIPT, @@ -462,6 +525,9 @@ private void queryState(String functionName, String key, int amount) "--key", key ); assertTrue(result.getStdout().contains("\"numberValue\": " + amount)); + assertTrue(result.getStdout().contains("\"version\": " + version)); + assertFalse(result.getStdout().contains("stringValue")); + assertFalse(result.getStdout().contains("byteValue")); } private void putAndQueryState(String functionName, String key, String state, String expect) @@ -488,6 +554,33 @@ private void putAndQueryState(String functionName, String key, String state, Str assertTrue(result.getStdout().contains(expect)); } + private void putAndQueryStateByte(String functionName, String key, String state, byte[] expect) + throws Exception { + container.execCmd( + PulsarCluster.ADMIN_SCRIPT, + "functions", + "putstate", + "--tenant", "public", + "--namespace", "default", + "--name", functionName, + "--state", state + ); + + ContainerExecResult result = container.execCmd( + PulsarCluster.ADMIN_SCRIPT, + "functions", + "querystate", + "--tenant", "public", + "--namespace", "default", + "--name", functionName, + "--key", key + ); + + FunctionState byteState = new Gson().fromJson(result.getStdout(), FunctionState.class); + assertNull(byteState.getStringValue()); + assertEquals(byteState.getByteValue(), expect); + } + private void publishAndConsumeMessages(String inputTopic, String outputTopic, int numMessages) throws Exception { diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java index 64a05503e1970..1f29e19f01873 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java @@ -89,7 +89,6 @@ public void setup() throws Exception { "org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder"); brokerEnvs.put("forceDeleteNamespaceAllowed", "true"); brokerEnvs.put("loadBalancerDebugModeEnabled", "true"); - brokerEnvs.put("topicLevelPoliciesEnabled", "false"); brokerEnvs.put("PULSAR_MEM", "-Xmx512M"); spec.brokerEnvs(brokerEnvs); pulsarCluster = PulsarCluster.forSpec(spec); @@ -125,6 +124,26 @@ public void startBroker() { brokerContainer.start(); } }); + String topicName = "persistent://" + DEFAULT_NAMESPACE + "/startBrokerCheck"; + Awaitility.await().atMost(120, TimeUnit.SECONDS).ignoreExceptions().until( + () -> { + for (BrokerContainer brokerContainer : pulsarCluster.getBrokers()) { + try (PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl( + brokerContainer.getHttpServiceUrl()).build()) { + if (admin.brokers().getActiveBrokers(clusterName).size() != NUM_BROKERS) { + return false; + } + try { + admin.topics().createPartitionedTopic(topicName, 10); + } catch (PulsarAdminException.ConflictException e) { + // expected + } + admin.lookups().lookupPartitionedTopic(topicName); + } + } + return true; + } + ); } } @@ -243,7 +262,7 @@ public void testDeleteNamespace() throws Exception { assertFalse(admin.namespaces().getNamespaces(DEFAULT_TENANT).contains(namespace)); } - @Test(timeOut = 40 * 1000) + @Test(timeOut = 120 * 1000) public void testStopBroker() throws Exception { String topicName = "persistent://" + DEFAULT_NAMESPACE + "/test-stop-broker-topic"; @@ -259,9 +278,11 @@ public void testStopBroker() throws Exception { } } - String broker1 = admin.lookups().lookupTopic(topicName); + Awaitility.waitAtMost(60, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> { + String broker1 = admin.lookups().lookupTopic(topicName); + assertNotEquals(broker1, broker); + }); - assertNotEquals(broker1, broker); } @Test(timeOut = 80 * 1000) @@ -301,7 +322,7 @@ public void testAntiaffinityPolicy() throws PulsarAdminException { assertEquals(result.size(), NUM_BROKERS); } - @Test(timeOut = 240 * 1000) + @Test(timeOut = 300 * 1000) public void testIsolationPolicy() throws Exception { final String namespaceIsolationPolicyName = "my-isolation-policy"; final String isolationEnabledNameSpace = DEFAULT_TENANT + "/my-isolation-policy" + nsSuffix; @@ -309,9 +330,10 @@ public void testIsolationPolicy() throws Exception { parameters1.put("min_limit", "1"); parameters1.put("usage_threshold", "100"); - Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted( + Awaitility.await().atMost(10, TimeUnit.SECONDS).ignoreExceptions().untilAsserted( () -> { - List activeBrokers = admin.brokers().getActiveBrokers(); + List activeBrokers = admin.brokers().getActiveBrokersAsync() + .get(5, TimeUnit.SECONDS); assertEquals(activeBrokers.size(), NUM_BROKERS); } ); @@ -350,15 +372,16 @@ public void testIsolationPolicy() throws Exception { } } - Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted( + Awaitility.await().atMost(60, TimeUnit.SECONDS).ignoreExceptions().untilAsserted( () -> { - List activeBrokers = admin.brokers().getActiveBrokers(); + List activeBrokers = admin.brokers().getActiveBrokersAsync() + .get(5, TimeUnit.SECONDS); assertEquals(activeBrokers.size(), 2); } ); - Awaitility.await().atMost(30, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> { - String ownerBroker = admin.lookups().lookupTopic(topic); + Awaitility.await().atMost(60, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> { + String ownerBroker = admin.lookups().lookupTopicAsync(topic).get(5, TimeUnit.SECONDS); assertEquals(extractBrokerIndex(ownerBroker), 1); }); @@ -369,20 +392,24 @@ public void testIsolationPolicy() throws Exception { } } - Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted( + Awaitility.await().atMost(60, TimeUnit.SECONDS).ignoreExceptions().untilAsserted( () -> { - List activeBrokers = admin.brokers().getActiveBrokers(); + List activeBrokers = admin.brokers().getActiveBrokersAsync().get(5, TimeUnit.SECONDS); assertEquals(activeBrokers.size(), 1); } ); - try { - admin.lookups().lookupTopic(topic); - fail(); - } catch (Exception ex) { - log.error("Failed to lookup topic: ", ex); - assertThat(ex.getMessage()).containsAnyOf("Failed to look up a broker", - "Failed to select the new owner broker for bundle"); - } + + Awaitility.await().atMost(60, TimeUnit.SECONDS).ignoreExceptions().untilAsserted( + () -> { + try { + admin.lookups().lookupTopicAsync(topic).get(5, TimeUnit.SECONDS); + } catch (Exception ex) { + log.error("Failed to lookup topic: ", ex); + assertThat(ex.getMessage()).contains("Failed to select the new owner broker for bundle"); + } + } + ); + } private void createNonPartitionedTopicAndRetry(String topicName) throws Exception { diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaTest.java index 8bb6de74c661d..d0421063b2d90 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaTest.java @@ -31,6 +31,8 @@ import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.tests.integration.schema.Schemas.Person; import org.apache.pulsar.tests.integration.schema.Schemas.PersonConsumeSchema; import org.apache.pulsar.tests.integration.schema.Schemas.Student; @@ -316,5 +318,14 @@ public void testPrimitiveSchemaTypeCompatibilityCheck() { } + @Test + public void testDeletePartitionedTopicWhenTopicReferenceIsNotReady() throws Exception { + final String topic = "persistent://public/default/tp-ref"; + admin.topics().createPartitionedTopic(topic, 20); + admin.schemas().createSchema(topic, + SchemaInfo.builder().type(SchemaType.STRING).schema(new byte[0]).build()); + admin.topics().deletePartitionedTopic(topic, false); + } + } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java index 88ff778732ed3..77f21df1a30c1 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java @@ -415,37 +415,18 @@ public synchronized void setupFunctionWorkers(String suffix, FunctionRuntimeType } private void startFunctionWorkersWithProcessContainerFactory(String suffix, int numFunctionWorkers) { - String serviceUrl = "pulsar://pulsar-broker-0:" + PulsarContainer.BROKER_PORT; - String httpServiceUrl = "http://pulsar-broker-0:" + PulsarContainer.BROKER_HTTP_PORT; workerContainers.putAll(runNumContainers( "functions-worker-process-" + suffix, numFunctionWorkers, - (name) -> new WorkerContainer(clusterName, name) - .withNetwork(network) - .withNetworkAliases(name) - // worker settings - .withEnv("PF_workerId", name) - .withEnv("PF_workerHostname", name) - .withEnv("PF_workerPort", "" + PulsarContainer.BROKER_HTTP_PORT) - .withEnv("PF_pulsarFunctionsCluster", clusterName) - .withEnv("PF_pulsarServiceUrl", serviceUrl) - .withEnv("PF_pulsarWebServiceUrl", httpServiceUrl) - // script - .withEnv("clusterName", clusterName) - .withEnv("zookeeperServers", ZKContainer.NAME) - // bookkeeper tools - .withEnv("zkServers", ZKContainer.NAME) + (name) -> createWorkerContainer(name) )); this.startWorkers(); } - private void startFunctionWorkersWithThreadContainerFactory(String suffix, int numFunctionWorkers) { + private WorkerContainer createWorkerContainer(String name) { String serviceUrl = "pulsar://pulsar-broker-0:" + PulsarContainer.BROKER_PORT; String httpServiceUrl = "http://pulsar-broker-0:" + PulsarContainer.BROKER_HTTP_PORT; - workerContainers.putAll(runNumContainers( - "functions-worker-thread-" + suffix, - numFunctionWorkers, - (name) -> new WorkerContainer(clusterName, name) + return new WorkerContainer(clusterName, name) .withNetwork(network) .withNetworkAliases(name) // worker settings @@ -455,13 +436,21 @@ private void startFunctionWorkersWithThreadContainerFactory(String suffix, int n .withEnv("PF_pulsarFunctionsCluster", clusterName) .withEnv("PF_pulsarServiceUrl", serviceUrl) .withEnv("PF_pulsarWebServiceUrl", httpServiceUrl) - .withEnv("PF_functionRuntimeFactoryClassName", "org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory") - .withEnv("PF_functionRuntimeFactoryConfigs_threadGroupName", "pf-container-group") // script .withEnv("clusterName", clusterName) .withEnv("zookeeperServers", ZKContainer.NAME) // bookkeeper tools - .withEnv("zkServers", ZKContainer.NAME) + .withEnv("zkServers", ZKContainer.NAME); + } + + private void startFunctionWorkersWithThreadContainerFactory(String suffix, int numFunctionWorkers) { + workerContainers.putAll(runNumContainers( + "functions-worker-thread-" + suffix, + numFunctionWorkers, + (name) -> createWorkerContainer(name) + .withEnv("PF_functionRuntimeFactoryClassName", + "org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory") + .withEnv("PF_functionRuntimeFactoryConfigs_threadGroupName", "pf-container-group") )); this.startWorkers(); } diff --git a/tests/integration/src/test/resources/pulsar-auth.xml b/tests/integration/src/test/resources/pulsar-auth.xml index 81d2c1361a269..3647730dee201 100644 --- a/tests/integration/src/test/resources/pulsar-auth.xml +++ b/tests/integration/src/test/resources/pulsar-auth.xml @@ -1,3 +1,4 @@ + - + - - + + - + diff --git a/tests/integration/src/test/resources/pulsar-backwards-compatibility.xml b/tests/integration/src/test/resources/pulsar-backwards-compatibility.xml index b50af74ed17f0..b0f55f3e04b16 100644 --- a/tests/integration/src/test/resources/pulsar-backwards-compatibility.xml +++ b/tests/integration/src/test/resources/pulsar-backwards-compatibility.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-cli.xml b/tests/integration/src/test/resources/pulsar-cli.xml index af55aca8a0098..1243dc7e485d7 100644 --- a/tests/integration/src/test/resources/pulsar-cli.xml +++ b/tests/integration/src/test/resources/pulsar-cli.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-function.xml b/tests/integration/src/test/resources/pulsar-function.xml index a18a65ff2e16d..03a8d83997a3c 100644 --- a/tests/integration/src/test/resources/pulsar-function.xml +++ b/tests/integration/src/test/resources/pulsar-function.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-io-ora-source.xml b/tests/integration/src/test/resources/pulsar-io-ora-source.xml index 1c5bb5faf677c..0a9927015be86 100644 --- a/tests/integration/src/test/resources/pulsar-io-ora-source.xml +++ b/tests/integration/src/test/resources/pulsar-io-ora-source.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-io-sinks.xml b/tests/integration/src/test/resources/pulsar-io-sinks.xml index 614c371f3e52c..046f977f3e13b 100644 --- a/tests/integration/src/test/resources/pulsar-io-sinks.xml +++ b/tests/integration/src/test/resources/pulsar-io-sinks.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-io-sources.xml b/tests/integration/src/test/resources/pulsar-io-sources.xml index 636b3e479195f..c995a7df249dd 100644 --- a/tests/integration/src/test/resources/pulsar-io-sources.xml +++ b/tests/integration/src/test/resources/pulsar-io-sources.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - \ No newline at end of file + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-loadbalance.xml b/tests/integration/src/test/resources/pulsar-loadbalance.xml index dfc4536e25592..d28dd1e0da7ee 100644 --- a/tests/integration/src/test/resources/pulsar-loadbalance.xml +++ b/tests/integration/src/test/resources/pulsar-loadbalance.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-messaging.xml b/tests/integration/src/test/resources/pulsar-messaging.xml index cfbdb22587034..f8813b614f5cb 100644 --- a/tests/integration/src/test/resources/pulsar-messaging.xml +++ b/tests/integration/src/test/resources/pulsar-messaging.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-plugin.xml b/tests/integration/src/test/resources/pulsar-plugin.xml index f88b67256e547..06fa8ee628c0f 100644 --- a/tests/integration/src/test/resources/pulsar-plugin.xml +++ b/tests/integration/src/test/resources/pulsar-plugin.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-process.xml b/tests/integration/src/test/resources/pulsar-process.xml index 8e5258d30624c..a6f1ac468d954 100644 --- a/tests/integration/src/test/resources/pulsar-process.xml +++ b/tests/integration/src/test/resources/pulsar-process.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-proxy.xml b/tests/integration/src/test/resources/pulsar-proxy.xml index ae6f13810535c..fabfb54d781a3 100644 --- a/tests/integration/src/test/resources/pulsar-proxy.xml +++ b/tests/integration/src/test/resources/pulsar-proxy.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-python.xml b/tests/integration/src/test/resources/pulsar-python.xml index a5faa6389e0f1..365768083d21b 100644 --- a/tests/integration/src/test/resources/pulsar-python.xml +++ b/tests/integration/src/test/resources/pulsar-python.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-schema.xml b/tests/integration/src/test/resources/pulsar-schema.xml index e07fdf2b2d86f..595bb8fda5521 100644 --- a/tests/integration/src/test/resources/pulsar-schema.xml +++ b/tests/integration/src/test/resources/pulsar-schema.xml @@ -1,3 +1,4 @@ + - + - - - - - - - + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-semantics.xml b/tests/integration/src/test/resources/pulsar-semantics.xml index 5b5402af4623b..69e171931f0e4 100644 --- a/tests/integration/src/test/resources/pulsar-semantics.xml +++ b/tests/integration/src/test/resources/pulsar-semantics.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-standalone.xml b/tests/integration/src/test/resources/pulsar-standalone.xml index d8892c0746181..45d79a72821b5 100644 --- a/tests/integration/src/test/resources/pulsar-standalone.xml +++ b/tests/integration/src/test/resources/pulsar-standalone.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-thread.xml b/tests/integration/src/test/resources/pulsar-thread.xml index cf3da15e8e1ad..dc1b8e3b3de15 100644 --- a/tests/integration/src/test/resources/pulsar-thread.xml +++ b/tests/integration/src/test/resources/pulsar-thread.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - - \ No newline at end of file + + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-tls.xml b/tests/integration/src/test/resources/pulsar-tls.xml index 153d14b62a725..88e90c43107fb 100644 --- a/tests/integration/src/test/resources/pulsar-tls.xml +++ b/tests/integration/src/test/resources/pulsar-tls.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-transaction.xml b/tests/integration/src/test/resources/pulsar-transaction.xml index 72c375d000d8e..425f58ebda639 100644 --- a/tests/integration/src/test/resources/pulsar-transaction.xml +++ b/tests/integration/src/test/resources/pulsar-transaction.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-upgrade.xml b/tests/integration/src/test/resources/pulsar-upgrade.xml index a52db54753372..cb312b04ed7a3 100644 --- a/tests/integration/src/test/resources/pulsar-upgrade.xml +++ b/tests/integration/src/test/resources/pulsar-upgrade.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-websocket.xml b/tests/integration/src/test/resources/pulsar-websocket.xml index 87bf832d4e40a..e5023451518e0 100644 --- a/tests/integration/src/test/resources/pulsar-websocket.xml +++ b/tests/integration/src/test/resources/pulsar-websocket.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/pulsar.xml b/tests/integration/src/test/resources/pulsar.xml index bdc5f27cc78fb..431502d15ea43 100644 --- a/tests/integration/src/test/resources/pulsar.xml +++ b/tests/integration/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + diff --git a/tests/integration/src/test/resources/tiered-filesystem-storage.xml b/tests/integration/src/test/resources/tiered-filesystem-storage.xml index b14077594e0c9..c8cb694ea2d02 100644 --- a/tests/integration/src/test/resources/tiered-filesystem-storage.xml +++ b/tests/integration/src/test/resources/tiered-filesystem-storage.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/tiered-jcloud-storage.xml b/tests/integration/src/test/resources/tiered-jcloud-storage.xml index f61f9f5fa1612..be699ba0a8680 100644 --- a/tests/integration/src/test/resources/tiered-jcloud-storage.xml +++ b/tests/integration/src/test/resources/tiered-jcloud-storage.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/pom.xml b/tests/pom.xml index 3a8d2ec8228be..50ff2eac257ab 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -1,4 +1,4 @@ - + - + pom 4.0.0 - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT - org.apache.pulsar.tests + io.streamnative.tests tests-parent Apache Pulsar :: Tests @@ -50,7 +49,7 @@ skipIntegrationTests - + integrationTests diff --git a/tests/pulsar-client-admin-shade-test/pom.xml b/tests/pulsar-client-admin-shade-test/pom.xml index f68ec9e9801b9..229879e5d2e0a 100644 --- a/tests/pulsar-client-admin-shade-test/pom.xml +++ b/tests/pulsar-client-admin-shade-test/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar.tests - tests-parent - 3.2.0-SNAPSHOT - - - pulsar-client-admin-shade-test - jar - Apache Pulsar :: Tests :: Pulsar-Client-Admin-Shade Test - - - - - org.apache.pulsar - pulsar-client-admin - ${project.version} - test - - - - org.apache.pulsar - pulsar-client-admin-api - ${project.version} - test - - - - org.testcontainers - testcontainers - test - - - - org.apache.pulsar - pulsar-client-messagecrypto-bc - ${project.version} - test - - - - - + + 4.0.0 + + io.streamnative.tests + tests-parent + 3.2.0-SNAPSHOT + + pulsar-client-admin-shade-test + jar + Apache Pulsar :: Tests :: Pulsar-Client-Admin-Shade Test + + + io.streamnative + pulsar-client-admin + ${project.version} + test + + + io.streamnative + pulsar-client-admin-api + ${project.version} + test + + + org.testcontainers + testcontainers + test + + + io.streamnative + pulsar-client-messagecrypto-bc + ${project.version} + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + ShadeTests + + + ShadeTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - ShadeTests - - - ShadeTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/pulsar.xml - - 1 - - - - - - + false + + src/test/resources/pulsar.xml + + 1 + + + + + + diff --git a/tests/pulsar-client-admin-shade-test/src/test/resources/pulsar.xml b/tests/pulsar-client-admin-shade-test/src/test/resources/pulsar.xml index d95dd95ae1775..2476755a95f7e 100644 --- a/tests/pulsar-client-admin-shade-test/src/test/resources/pulsar.xml +++ b/tests/pulsar-client-admin-shade-test/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - + + + + + + diff --git a/tests/pulsar-client-all-shade-test/pom.xml b/tests/pulsar-client-all-shade-test/pom.xml index 427ef8ae64c30..9f02a24d68845 100644 --- a/tests/pulsar-client-all-shade-test/pom.xml +++ b/tests/pulsar-client-all-shade-test/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar.tests - tests-parent - 3.2.0-SNAPSHOT - - - pulsar-client-all-shade-test - jar - Apache Pulsar :: Tests :: Pulsar-Client-All-Shade Test - - - - - org.apache.pulsar - pulsar-client-all - ${project.version} - test - - - - org.testcontainers - testcontainers - test - - - - org.apache.pulsar - bouncy-castle-bc - ${project.version} - pkg - - - - org.apache.pulsar - pulsar-client-messagecrypto-bc - ${project.version} - - - - - + + 4.0.0 + + io.streamnative.tests + tests-parent + 3.2.0-SNAPSHOT + + pulsar-client-all-shade-test + jar + Apache Pulsar :: Tests :: Pulsar-Client-All-Shade Test + + + io.streamnative + pulsar-client-all + ${project.version} + test + + + org.testcontainers + testcontainers + test + + + io.streamnative + bouncy-castle-bc + ${project.version} + pkg + + + io.streamnative + pulsar-client-messagecrypto-bc + ${project.version} + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + ShadeTests + + + ShadeTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - ShadeTests - - - ShadeTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/pulsar.xml - - 1 - - - - - - + false + + src/test/resources/pulsar.xml + + 1 + + + + + + diff --git a/tests/pulsar-client-all-shade-test/src/test/resources/pulsar.xml b/tests/pulsar-client-all-shade-test/src/test/resources/pulsar.xml index 5c725f80eaea6..cfbc5ce59488b 100644 --- a/tests/pulsar-client-all-shade-test/src/test/resources/pulsar.xml +++ b/tests/pulsar-client-all-shade-test/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - + + + + + + diff --git a/tests/pulsar-client-shade-test/pom.xml b/tests/pulsar-client-shade-test/pom.xml index d1694c3f6b307..c53f88ae61aa6 100644 --- a/tests/pulsar-client-shade-test/pom.xml +++ b/tests/pulsar-client-shade-test/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar.tests - tests-parent - 3.2.0-SNAPSHOT - - - pulsar-client-shade-test - jar - Apache Pulsar :: Tests :: Pulsar-Client-Shade Test - - - - - org.apache.pulsar - pulsar-client - ${project.version} - test - - - - org.apache.pulsar - pulsar-client-admin - ${project.version} - test - - - - org.testcontainers - testcontainers - test - - - - - + + 4.0.0 + + io.streamnative.tests + tests-parent + 3.2.0-SNAPSHOT + + pulsar-client-shade-test + jar + Apache Pulsar :: Tests :: Pulsar-Client-Shade Test + + + io.streamnative + pulsar-client + ${project.version} + test + + + io.streamnative + pulsar-client-admin + ${project.version} + test + + + org.testcontainers + testcontainers + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + ShadeTests + + + ShadeTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - ShadeTests - - - ShadeTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/pulsar.xml - - 1 - - - - - - + false + + src/test/resources/pulsar.xml + + 1 + + + + + + diff --git a/tests/pulsar-client-shade-test/src/test/resources/pulsar.xml b/tests/pulsar-client-shade-test/src/test/resources/pulsar.xml index 6132ad05f6ccf..e2ac6a9ee4f30 100644 --- a/tests/pulsar-client-shade-test/src/test/resources/pulsar.xml +++ b/tests/pulsar-client-shade-test/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - + + + + + + diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index fcaf83be95113..2e96e1110e3d6 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - - org.apache.pulsar - tiered-storage-parent - 3.2.0-SNAPSHOT - .. - - - tiered-storage-file-system - Apache Pulsar :: Tiered Storage :: File System - - - ${project.groupId} - managed-ledger - ${project.version} - - - org.apache.hadoop - hadoop-common - ${hdfs-offload-version3} - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - - - org.apache.hadoop - hadoop-hdfs-client - ${hdfs-offload-version3} - - - org.apache.avro - avro - - - org.mortbay.jetty - jetty - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-server - - - javax.servlet - servlet-api - - - - - - org.apache.avro - avro - ${avro.version} - - - - net.minidev - json-smart - ${json-smart.version} - - - com.google.protobuf - protobuf-java - - - - ${project.groupId} - testmocks - ${project.version} - test - - - - org.apache.hadoop - hadoop-minicluster - ${hdfs-offload-version3} - test - - - io.netty - netty-all - - - org.bouncycastle - * - - - - - - org.bouncycastle - bcpkix-jdk18on - test - - - + + 4.0.0 + + io.streamnative + tiered-storage-parent + 3.2.0-SNAPSHOT + .. + + tiered-storage-file-system + Apache Pulsar :: Tiered Storage :: File System + + + ${project.groupId} + managed-ledger + ${project.version} + + + org.apache.hadoop + hadoop-common + ${hdfs-offload-version3} + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.hadoop + hadoop-hdfs-client + ${hdfs-offload-version3} + + + org.apache.avro + avro + + + org.mortbay.jetty + jetty + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + javax.servlet + servlet-api + + + + + + org.apache.avro + avro + ${avro.version} + + + + net.minidev + json-smart + ${json-smart.version} + + + com.google.protobuf + protobuf-java + + + ${project.groupId} + testmocks + ${project.version} + test + + + org.apache.hadoop + hadoop-minicluster + ${hdfs-offload-version3} + test + + io.netty - netty-codec-http - - - - org.eclipse.jetty - jetty-server - test - - - org.eclipse.jetty - jetty-alpn-conscrypt-server - test - - - org.eclipse.jetty - jetty-servlet - test - - - org.eclipse.jetty - jetty-util - test - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - spotbugs - verify - - check - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - - - - - owasp-dependency-check - - - - org.owasp - dependency-check-maven - ${dependency-check-maven.version} - - - - aggregate - - none - - - - - - - + + owasp-dependency-check + + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + + + aggregate + + none + + + + + + + diff --git a/tiered-storage/file-system/src/main/resources/findbugsExclude.xml b/tiered-storage/file-system/src/main/resources/findbugsExclude.xml index 051f0e68e257a..1b8cc1ac799ca 100644 --- a/tiered-storage/file-system/src/main/resources/findbugsExclude.xml +++ b/tiered-storage/file-system/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative tiered-storage-parent 3.2.0-SNAPSHOT .. - tiered-storage-jcloud Apache Pulsar :: Tiered Storage :: JCloud - ${project.groupId} managed-ledger ${project.version} - ${project.groupId} jclouds-shaded @@ -74,7 +70,6 @@ - org.apache.jclouds jclouds-allblobstore @@ -85,12 +80,10 @@ com.amazonaws aws-java-sdk-core - com.amazonaws aws-java-sdk-sts - ${project.groupId} testmocks @@ -103,7 +96,6 @@ ${jclouds.version} provided - javax.xml.bind jaxb-api @@ -115,13 +107,11 @@ runtime - com.sun.activation javax.activation runtime - @@ -129,7 +119,6 @@ org.apache.nifi nifi-nar-maven-plugin - com.github.spotbugs spotbugs-maven-plugin diff --git a/tiered-storage/jcloud/src/main/resources/findbugsExclude.xml b/tiered-storage/jcloud/src/main/resources/findbugsExclude.xml index a8a9b9aae925f..c6ad70823a5ac 100644 --- a/tiered-storage/jcloud/src/main/resources/findbugsExclude.xml +++ b/tiered-storage/jcloud/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar 3.2.0-SNAPSHOT .. - tiered-storage-parent Apache Pulsar :: Tiered Storage :: Parent - ${project.version} - jcloud file-system