Skip to content

Commit

Permalink
HS-660 Align kafka processors.
Browse files Browse the repository at this point in the history
Signed-off-by: Łukasz Dywicki <luke@code-house.org>
  • Loading branch information
splatch committed Jul 4, 2023
1 parent 34d9571 commit 0792289
Show file tree
Hide file tree
Showing 13 changed files with 588 additions and 353 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -28,50 +28,32 @@

package org.opennms.miniongateway.grpc.server.flows;

import lombok.RequiredArgsConstructor;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.opennms.horizon.flows.document.FlowDocumentLog;
import org.opennms.horizon.flows.document.TenantLocationSpecificFlowDocumentLog;
import org.opennms.horizon.shared.flows.mapper.TenantLocationSpecificFlowDocumentLogMapper;
import org.opennms.horizon.shared.grpc.common.LocationServerInterceptor;
import org.opennms.horizon.shared.grpc.common.TenantIDGrpcServerInterceptor;
import org.opennms.horizon.shared.ipc.sink.api.MessageConsumer;
import org.opennms.horizon.shared.ipc.sink.api.SinkModule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.opennms.miniongateway.grpc.server.kafka.SinkMessageKafkaPublisher;
import org.opennms.miniongateway.grpc.server.kafka.SinkMessageKafkaPublisherFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Component;

/**
* Forwarder of Flow messages - received via GRPC and forwarded to Kafka.
*/
@Component
public class FlowKafkaForwarder implements MessageConsumer<FlowDocumentLog, FlowDocumentLog> {
public static final String DEFAULT_TASK_RESULTS_TOPIC = "flows";

private final Logger logger = LoggerFactory.getLogger(FlowKafkaForwarder.class);

private final TenantIDGrpcServerInterceptor tenantIDGrpcInterceptor;

private final LocationServerInterceptor locationServerInterceptor;

private final KafkaTemplate<String, byte[]> kafkaTemplate;

private final TenantLocationSpecificFlowDocumentLogMapper tenantLocationSpecificFlowDocumentLogMapper;

@Value("${task.results.kafka-topic:" + DEFAULT_TASK_RESULTS_TOPIC + "}")
private String kafkaTopic;

public FlowKafkaForwarder(@Autowired TenantIDGrpcServerInterceptor tenantIDGrpcInterceptor,
@Autowired LocationServerInterceptor locationServerInterceptor,
@Autowired KafkaTemplate<String, byte[]> kafkaTemplate,
@Autowired TenantLocationSpecificFlowDocumentLogMapper tenantLocationSpecificFlowDocumentLogMapper) {
this.tenantIDGrpcInterceptor = tenantIDGrpcInterceptor;
this.locationServerInterceptor = locationServerInterceptor;
this.kafkaTemplate = kafkaTemplate;
this.tenantLocationSpecificFlowDocumentLogMapper = tenantLocationSpecificFlowDocumentLogMapper;
public static final String DEFAULT_FLOW_RESULTS_TOPIC = "flows";
private final SinkMessageKafkaPublisher<FlowDocumentLog, TenantLocationSpecificFlowDocumentLog> kafkaPublisher;

@Autowired
public FlowKafkaForwarder(SinkMessageKafkaPublisherFactory messagePublisherFactory, TenantLocationSpecificFlowDocumentLogMapper mapper,
@Value("${flow.results.kafka-topic:" + DEFAULT_FLOW_RESULTS_TOPIC + "}") String kafkaTopic) {
this.kafkaPublisher = messagePublisherFactory.create(
mapper::mapBareToTenanted,
kafkaTopic
);
}

@Override
Expand All @@ -81,27 +63,7 @@ public SinkModule<FlowDocumentLog, FlowDocumentLog> getModule() {

@Override
public void handleMessage(FlowDocumentLog messageLog) {
// Retrieve the Tenant ID from the TenantID GRPC Interceptor
String tenantId = tenantIDGrpcInterceptor.readCurrentContextTenantId();
// Ditto for location
String locationId = locationServerInterceptor.readCurrentContextLocationId();
logger.trace("Received flow; sending to Kafka: tenantId: {}; locationId={}; kafka-topic={}; message={}", tenantId, locationId, kafkaTopic, messageLog);


var tenantLocationSpecificFlowDocumentLog =
tenantLocationSpecificFlowDocumentLogMapper.mapBareToTenanted(tenantId, locationId, messageLog);

sendToKafka(tenantLocationSpecificFlowDocumentLog);
this.kafkaPublisher.send(messageLog);
}

//========================================
// INTERNALS
//----------------------------------------

private void sendToKafka(TenantLocationSpecificFlowDocumentLog tenantLocationSpecificFlowDocumentLog) {
byte[] rawContent = tenantLocationSpecificFlowDocumentLog.toByteArray();
var producerRecord = new ProducerRecord<String, byte[]>(kafkaTopic, rawContent);

this.kafkaTemplate.send(producerRecord);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -28,68 +28,41 @@

package org.opennms.miniongateway.grpc.server.heartbeat;

import org.apache.kafka.clients.producer.ProducerRecord;
import org.opennms.horizon.grpc.heartbeat.contract.HeartbeatMessage;
import org.opennms.horizon.grpc.heartbeat.contract.TenantLocationSpecificHeartbeatMessage;
import org.opennms.horizon.grpc.heartbeat.contract.mapper.TenantLocationSpecificHeartbeatMessageMapper;
import org.opennms.horizon.shared.grpc.common.LocationServerInterceptor;
import org.opennms.horizon.shared.grpc.common.TenantIDGrpcServerInterceptor;
import org.opennms.horizon.shared.ipc.sink.api.MessageConsumer;
import org.opennms.horizon.shared.ipc.sink.api.SinkModule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.opennms.miniongateway.grpc.server.kafka.SinkMessageKafkaPublisher;
import org.opennms.miniongateway.grpc.server.kafka.SinkMessageKafkaPublisherFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Component;

import io.opentelemetry.api.trace.Span;

/**
* Forwarder of Heartbeat messages - received via GRPC and forwarded to Kafka.
*/
@Component
public class HeartbeatKafkaForwarder implements MessageConsumer<HeartbeatMessage, HeartbeatMessage> {
public static final String DEFAULT_TASK_RESULTS_TOPIC = "heartbeat";

private final Logger logger = LoggerFactory.getLogger(HeartbeatKafkaForwarder.class);

@Autowired
private KafkaTemplate<String, byte[]> kafkaTemplate;

@Value("${task.results.kafka-topic:" + DEFAULT_TASK_RESULTS_TOPIC + "}")
private String kafkaTopic;

@Autowired
private TenantIDGrpcServerInterceptor tenantIDGrpcInterceptor;
public static final String DEFAULT_HEARTBEAT_RESULTS_TOPIC = "heartbeat";
private final SinkMessageKafkaPublisher<HeartbeatMessage, TenantLocationSpecificHeartbeatMessage> kafkaPublisher;

@Autowired
private LocationServerInterceptor locationServerInterceptor;

@Autowired
private TenantLocationSpecificHeartbeatMessageMapper tenantLocationSpecificHeartbeatMessageMapper;
public HeartbeatKafkaForwarder(SinkMessageKafkaPublisherFactory messagePublisherFactory, TenantLocationSpecificHeartbeatMessageMapper mapper,
@Value("${heartbeat.results.kafka-topic:" + DEFAULT_HEARTBEAT_RESULTS_TOPIC + "}") String kafkaTopic) {
this.kafkaPublisher = messagePublisherFactory.create(
mapper::mapBareToTenanted,
kafkaTopic
);
}

@Override
public SinkModule<HeartbeatMessage, HeartbeatMessage> getModule() {
return new HeartbeatModule();
}

@Override
public void handleMessage(HeartbeatMessage heartbeatMessage) {
// Retrieve the Tenant ID from the TenantID GRPC Interceptor
String tenantId = tenantIDGrpcInterceptor.readCurrentContextTenantId();
// And the locationId from its Interceptor
String locationId = locationServerInterceptor.readCurrentContextLocationId();

logger.info("Received heartbeat; sending to Kafka: tenantId={}; locationId={}; kafka-topic={}; message={}", tenantId, locationId, kafkaTopic, heartbeatMessage);
Span.current().setAttribute("message", heartbeatMessage.toString());

TenantLocationSpecificHeartbeatMessage mapped = tenantLocationSpecificHeartbeatMessageMapper.mapBareToTenanted(tenantId, locationId, heartbeatMessage);

byte[] rawContent = mapped.toByteArray();
var producerRecord = new ProducerRecord<String, byte[]>(kafkaTopic, rawContent);

this.kafkaTemplate.send(producerRecord);
public void handleMessage(HeartbeatMessage message) {
this.kafkaPublisher.send(message);
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
/*******************************************************************************
* This file is part of OpenNMS(R).
*
* Copyright (C) 2022 The OpenNMS Group, Inc.
* OpenNMS(R) is Copyright (C) 1999-2022 The OpenNMS Group, Inc.
*
* OpenNMS(R) is a registered trademark of The OpenNMS Group, Inc.
*
* OpenNMS(R) is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License,
* or (at your option) any later version.
*
* OpenNMS(R) is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with OpenNMS(R). If not, see:
* http://www.gnu.org/licenses/
*
* For more information contact:
* OpenNMS(R) Licensing <license@opennms.org>
* http://www.opennms.org/
* http://www.opennms.com/
*******************************************************************************/

package org.opennms.miniongateway.grpc.server.kafka;

import com.google.protobuf.Message;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.opennms.horizon.shared.grpc.common.LocationServerInterceptor;
import org.opennms.horizon.shared.grpc.common.TenantIDGrpcServerInterceptor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.kafka.core.KafkaTemplate;

/**
* A helper class which produces kafka messages.
*
* It additionally retrieves tenant information from present context.
* @param <I> Input message (grpc side) kind
* @param <O> Output message (kafka side) type
*/
public class SinkMessageKafkaPublisher<I extends Message, O extends Message> {

private final Logger logger = LoggerFactory.getLogger(SinkMessageKafkaPublisher.class);
private final KafkaTemplate<String, byte[]> kafkaTemplate;
private final TenantIDGrpcServerInterceptor tenantInterceptor;
private final LocationServerInterceptor locationInterceptor;
private final SinkMessageMapper<I, O> mapper;
private final String topic;

public SinkMessageKafkaPublisher(KafkaTemplate<String, byte[]> kafkaTemplate,
TenantIDGrpcServerInterceptor tenantInterceptor, LocationServerInterceptor locationInterceptor,
SinkMessageMapper<I, O> mapper, String topic) {
this.kafkaTemplate = kafkaTemplate;
this.tenantInterceptor = tenantInterceptor;
this.locationInterceptor = locationInterceptor;
this.mapper = mapper;
this.topic = topic;
}

/**
* Map passed In message to a backend message which is then used as a payload for record sent to Kafka.
*
* @param message content to include as the message payload.
*/
public void send(I message) {
String tenantId = tenantInterceptor.readCurrentContextTenantId();
String locationId = locationInterceptor.readCurrentContextLocationId();

O mapped = mapper.map(tenantId, locationId, message);
logger.trace("Received {}; sending a {} to kafka topic {}; tenantId: {}; locationId={}; message={}",
message.getDescriptorForType().getName(), mapped.getDescriptorForType().getName(), topic, tenantId, locationId, mapped);

kafkaTemplate.send(new ProducerRecord<>(topic, mapped.toByteArray()));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
/*******************************************************************************
* This file is part of OpenNMS(R).
*
* Copyright (C) 2023 The OpenNMS Group, Inc.
* OpenNMS(R) is Copyright (C) 1999-2023 The OpenNMS Group, Inc.
*
* OpenNMS(R) is a registered trademark of The OpenNMS Group, Inc.
*
* OpenNMS(R) is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License,
* or (at your option) any later version.
*
* OpenNMS(R) is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with OpenNMS(R). If not, see:
* http://www.gnu.org/licenses/
*
* For more information contact:
* OpenNMS(R) Licensing <license@opennms.org>
* http://www.opennms.org/
* http://www.opennms.com/
*******************************************************************************/

package org.opennms.miniongateway.grpc.server.kafka;

import com.google.protobuf.Message;
import lombok.RequiredArgsConstructor;
import org.opennms.horizon.shared.grpc.common.LocationServerInterceptor;
import org.opennms.horizon.shared.grpc.common.TenantIDGrpcServerInterceptor;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Component;

@Component
@RequiredArgsConstructor
public class SinkMessageKafkaPublisherFactory {

private final TenantIDGrpcServerInterceptor tenantInterceptor;
private final LocationServerInterceptor locationInterceptor;
private final KafkaTemplate<String, byte[]> kafkaTemplate;

public <I extends Message, O extends Message> SinkMessageKafkaPublisher<I, O> create(SinkMessageMapper<I, O> mapper, String topic) {
return new SinkMessageKafkaPublisher<>(kafkaTemplate, tenantInterceptor, locationInterceptor, mapper, topic);
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/*******************************************************************************
* This file is part of OpenNMS(R).
*
* Copyright (C) 2023 The OpenNMS Group, Inc.
* OpenNMS(R) is Copyright (C) 1999-2023 The OpenNMS Group, Inc.
*
* OpenNMS(R) is a registered trademark of The OpenNMS Group, Inc.
*
* OpenNMS(R) is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License,
* or (at your option) any later version.
*
* OpenNMS(R) is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with OpenNMS(R). If not, see:
* http://www.gnu.org/licenses/
*
* For more information contact:
* OpenNMS(R) Licensing <license@opennms.org>
* http://www.opennms.org/
* http://www.opennms.com/
*******************************************************************************/

package org.opennms.miniongateway.grpc.server.kafka;

import com.google.protobuf.Message;

/**
* Mapper for messages sent by minion via sink channels.
*
* @param <I> Minion message.
* @param <O> Backend message.
*/
public interface SinkMessageMapper<I extends Message, O extends Message> {

/**
* Maps given message into backend message.
*
* @param tenantId Tenant for which message was generated.
* @param locationId Location for which message was generated.
* @param message Message sent by minion.
* @return Mapped message which should embed both tenantId and locationId.
*/
O map(String tenantId, String locationId, I message);

}
Loading

0 comments on commit 0792289

Please sign in to comment.