-
Notifications
You must be signed in to change notification settings - Fork 35
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #27 from KonstantinCodes/feature/refactoring
Refactoring
- Loading branch information
Showing
17 changed files
with
519 additions
and
170 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| #! /bin/bash | ||
| end=$((SECONDS+60)) | ||
|
|
||
| while [ $SECONDS -lt $end ]; do | ||
| if echo dump | nc localhost 2181 | grep broker ; then | ||
| exit 0 | ||
| else | ||
| echo "Kafka didn't start in time" | ||
| fi | ||
| sleep 1 | ||
| done |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,136 @@ | ||
| <?php | ||
|
|
||
| declare(strict_types=1); | ||
|
|
||
| namespace Koco\Kafka\Messenger; | ||
|
|
||
| use Koco\Kafka\RdKafka\RdKafkaFactory; | ||
| use Psr\Log\LoggerInterface; | ||
| use RdKafka\KafkaConsumer; | ||
| use Symfony\Component\Messenger\Envelope; | ||
| use Symfony\Component\Messenger\Exception\TransportException; | ||
| use Symfony\Component\Messenger\Transport\Receiver\ReceiverInterface; | ||
| use Symfony\Component\Messenger\Transport\Serialization\SerializerInterface; | ||
|
|
||
| class KafkaReceiver implements ReceiverInterface | ||
| { | ||
| /** @var LoggerInterface */ | ||
| private $logger; | ||
|
|
||
| /** @var SerializerInterface */ | ||
| private $serializer; | ||
|
|
||
| /** @var RdKafkaFactory */ | ||
| private $rdKafkaFactory; | ||
|
|
||
| /** @var KafkaReceiverProperties */ | ||
| private $properties; | ||
|
|
||
| /** @var KafkaConsumer */ | ||
| private $consumer; | ||
|
|
||
| /** @var bool */ | ||
| private $subscribed; | ||
|
|
||
| public function __construct( | ||
| LoggerInterface $logger, | ||
| SerializerInterface $serializer, | ||
| RdKafkaFactory $rdKafkaFactory, | ||
| KafkaReceiverProperties $properties | ||
| ) { | ||
| $this->logger = $logger; | ||
| $this->serializer = $serializer; | ||
| $this->rdKafkaFactory = $rdKafkaFactory; | ||
| $this->properties = $properties; | ||
|
|
||
| $this->subscribed = false; | ||
| } | ||
|
|
||
| public function get(): iterable | ||
| { | ||
| $message = $this->getSubscribedConsumer()->consume($this->properties->getReceiveTimeoutMs()); | ||
|
|
||
| switch ($message->err) { | ||
| case RD_KAFKA_RESP_ERR_NO_ERROR: | ||
| $this->logger->info(sprintf( | ||
| 'Kafka: Message %s %s %s received ', | ||
| $message->topic_name, | ||
| $message->partition, | ||
| $message->offset | ||
| )); | ||
|
|
||
| $envelope = $this->serializer->decode([ | ||
| 'body' => $message->payload, | ||
| 'headers' => $message->headers, | ||
| ]); | ||
|
|
||
| return [$envelope->with(new KafkaMessageStamp($message))]; | ||
| case RD_KAFKA_RESP_ERR__PARTITION_EOF: | ||
| $this->logger->info('Kafka: Partition EOF reached. Waiting for next message ...'); | ||
| break; | ||
| case RD_KAFKA_RESP_ERR__TIMED_OUT: | ||
| $this->logger->debug('Kafka: Consumer timeout.'); | ||
| break; | ||
| case RD_KAFKA_RESP_ERR__TRANSPORT: | ||
| $this->logger->debug('Kafka: Broker transport failure.'); | ||
| break; | ||
| default: | ||
| throw new TransportException($message->errstr(), $message->err); | ||
| } | ||
|
|
||
| return []; | ||
| } | ||
|
|
||
| public function ack(Envelope $envelope): void | ||
| { | ||
| $consumer = $this->getConsumer(); | ||
|
|
||
| /** @var KafkaMessageStamp $transportStamp */ | ||
| $transportStamp = $envelope->last(KafkaMessageStamp::class); | ||
| $message = $transportStamp->getMessage(); | ||
|
|
||
| if ($this->properties->isCommitAsync()) { | ||
| $consumer->commitAsync($message); | ||
|
|
||
| $this->logger->info(sprintf( | ||
| 'Offset topic=%s partition=%s offset=%s to be committed asynchronously.', | ||
| $message->topic_name, | ||
| $message->partition, | ||
| $message->offset | ||
| )); | ||
| } else { | ||
| $consumer->commit($message); | ||
|
|
||
| $this->logger->info(sprintf( | ||
| 'Offset topic=%s partition=%s offset=%s successfully committed.', | ||
| $message->topic_name, | ||
| $message->partition, | ||
| $message->offset | ||
| )); | ||
| } | ||
| } | ||
|
|
||
| public function reject(Envelope $envelope): void | ||
| { | ||
| // Do nothing. auto commit should be set to false! | ||
| } | ||
|
|
||
| private function getSubscribedConsumer(): KafkaConsumer | ||
| { | ||
| $consumer = $this->getConsumer(); | ||
|
|
||
| if (false === $this->subscribed) { | ||
| $this->logger->info('Partition assignment...'); | ||
| $consumer->subscribe([$this->properties->getTopicName()]); | ||
|
|
||
| $this->subscribed = true; | ||
| } | ||
|
|
||
| return $consumer; | ||
| } | ||
|
|
||
| private function getConsumer(): KafkaConsumer | ||
| { | ||
| return $this->consumer ?? $this->consumer = $this->rdKafkaFactory->createConsumer($this->properties->getKafkaConf()); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| <?php | ||
|
|
||
| declare(strict_types=1); | ||
|
|
||
| namespace Koco\Kafka\Messenger; | ||
|
|
||
| use RdKafka\Conf as KafkaConf; | ||
|
|
||
| final class KafkaReceiverProperties | ||
| { | ||
| /** @var KafkaConf */ | ||
| private $kafkaConf; | ||
|
|
||
| /** @var string */ | ||
| private $topicName; | ||
|
|
||
| /** @var int */ | ||
| private $receiveTimeoutMs; | ||
|
|
||
| /** @var bool */ | ||
| private $commitAsync; | ||
|
|
||
| public function __construct( | ||
| KafkaConf $kafkaConf, | ||
| string $topicName, | ||
| int $receiveTimeoutMs, | ||
| bool $commitAsync | ||
| ) { | ||
| $this->kafkaConf = $kafkaConf; | ||
| $this->topicName = $topicName; | ||
| $this->receiveTimeoutMs = $receiveTimeoutMs; | ||
| $this->commitAsync = $commitAsync; | ||
| } | ||
|
|
||
| public function getKafkaConf(): KafkaConf | ||
| { | ||
| return $this->kafkaConf; | ||
| } | ||
|
|
||
| public function getTopicName(): string | ||
| { | ||
| return $this->topicName; | ||
| } | ||
|
|
||
| public function getReceiveTimeoutMs(): int | ||
| { | ||
| return $this->receiveTimeoutMs; | ||
| } | ||
|
|
||
| public function isCommitAsync(): bool | ||
| { | ||
| return $this->commitAsync; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| <?php | ||
|
|
||
| declare(strict_types=1); | ||
|
|
||
| namespace Koco\Kafka\Messenger; | ||
|
|
||
| use Koco\Kafka\RdKafka\RdKafkaFactory; | ||
| use Psr\Log\LoggerInterface; | ||
| use RdKafka\Producer as KafkaProducer; | ||
| use Symfony\Component\Messenger\Envelope; | ||
| use Symfony\Component\Messenger\Exception\TransportException; | ||
| use Symfony\Component\Messenger\Transport\Sender\SenderInterface; | ||
| use Symfony\Component\Messenger\Transport\Serialization\SerializerInterface; | ||
|
|
||
| class KafkaSender implements SenderInterface | ||
| { | ||
| /** @var LoggerInterface */ | ||
| private $logger; | ||
|
|
||
| /** @var SerializerInterface */ | ||
| private $serializer; | ||
|
|
||
| /** @var RdKafkaFactory */ | ||
| private $rdKafkaFactory; | ||
|
|
||
| /** @var KafkaSenderProperties */ | ||
| private $properties; | ||
|
|
||
| /** @var KafkaProducer */ | ||
| private $producer; | ||
|
|
||
| public function __construct( | ||
| LoggerInterface $logger, | ||
| SerializerInterface $serializer, | ||
| RdKafkaFactory $rdKafkaFactory, | ||
| KafkaSenderProperties $properties | ||
| ) { | ||
| $this->logger = $logger; | ||
| $this->serializer = $serializer; | ||
| $this->rdKafkaFactory = $rdKafkaFactory; | ||
| $this->properties = $properties; | ||
| } | ||
|
|
||
| public function send(Envelope $envelope): Envelope | ||
| { | ||
| $producer = $this->getProducer(); | ||
| $topic = $producer->newTopic($this->properties->getTopicName()); | ||
|
|
||
| $payload = $this->serializer->encode($envelope); | ||
|
|
||
| $topic->producev( | ||
| RD_KAFKA_PARTITION_UA, | ||
| 0, | ||
| $payload['body'], | ||
| $payload['key'] ?? null, | ||
| $payload['headers'] ?? null, | ||
| $payload['timestamp_ms'] ?? null | ||
| ); | ||
|
|
||
| for ($flushRetries = 0; $flushRetries < $this->properties->getFlushRetries() + 1; ++$flushRetries) { | ||
| $code = $producer->flush($this->properties->getFlushTimeoutMs()); | ||
| if ($code === RD_KAFKA_RESP_ERR_NO_ERROR) { | ||
| $this->logger->info(sprintf('Kafka message sent%s', \array_key_exists('key', $payload) ? ' with key ' . $payload['key'] : '')); | ||
| break; | ||
| } | ||
| } | ||
|
|
||
| if ($code !== RD_KAFKA_RESP_ERR_NO_ERROR) { | ||
| throw new TransportException('Kafka producer response error: ' . $code, $code); | ||
| } | ||
|
|
||
| return $envelope; | ||
| } | ||
|
|
||
| private function getProducer(): KafkaProducer | ||
| { | ||
| return $this->producer ?? $this->producer = $this->rdKafkaFactory->createProducer($this->properties->getKafkaConf()); | ||
| } | ||
| } |
Oops, something went wrong.