Skip to content

Commit

Permalink
Fixed caps retrieval timing and added audio sending capabilities
Browse files Browse the repository at this point in the history
  • Loading branch information
@LufeBisect
LufeBisect committed Dec 26, 2024
1 parent 21da41d commit 8a6cf99
Show file tree
Hide file tree
Showing 5 changed files with 203 additions and 72 deletions.
2 changes: 1 addition & 1 deletion cpp/demos/config/nmos_plugin_node_config.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,4 @@
"device": {},
"receivers": [],
"senders": []
}
}
154 changes: 107 additions & 47 deletions cpp/libs/gst_nmos_sender_plugin/gst_nmos_sender_plugin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,9 @@ GST_DEBUG_CATEGORY_STATIC(gst_nmossender_debug_category);
typedef struct _GstNmossender
{
GstBin parent;
GstElement* payloader;
GstElement* queue;
GstElement* video_payloader;
GstElement* audio_payloader;
GstElement* udpsink;
ossrf::nmos_client_uptr client;
GstCaps* caps;
Expand Down Expand Up @@ -69,10 +71,14 @@ G_DEFINE_TYPE_WITH_CODE(GstNmossender, gst_nmossender, GST_TYPE_BIN,
GST_DEBUG_CATEGORY_INIT(gst_nmossender_debug_category, "nmossender", 0, "NMOS Sender Plugin"))

/* Pad template */
static GstStaticPadTemplate sink_template =
GST_STATIC_PAD_TEMPLATE("sink", GST_PAD_SINK, GST_PAD_REQUEST,
GST_STATIC_CAPS("video/x-raw, "
"format=(string){ I420, NV12, RGB } "));
static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE("sink", GST_PAD_SINK, GST_PAD_REQUEST,
GST_STATIC_CAPS("video/x-raw, "
"format=(string){ UYVP }; "
"audio/x-raw, "
"format=(string)S24BE, "
"layout=(string)interleaved, "
"rate=(int)[ 1, 2147483647 ], "
"channels=(int)[ 1, 2147483647 ]"));

/* Set properties so element variables can change depending on them */
static void gst_nmossender_set_property(GObject* object, guint property_id, const GValue* value, GParamSpec* pspec)
Expand Down Expand Up @@ -139,9 +145,8 @@ static void gst_nmossender_set_property(GObject* object, guint property_id, cons
break;

case PropertyId::DestinationPort:
g_free(self->config.network.destination_port);
self->config.network.destination_port = g_value_dup_string(value);
g_object_set(G_OBJECT(self->udpsink), "port", atoi(self->config.network.destination_port), NULL);
self->config.network.destination_port = atoi(g_value_dup_string(value));
g_object_set(G_OBJECT(self->udpsink), "port", self->config.network.destination_port, NULL);
break;

default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); break;
Expand All @@ -165,7 +170,7 @@ static void gst_nmossender_get_property(GObject* object, guint property_id, GVal
case PropertyId::SourceAddress: g_value_set_string(value, self->config.network.source_address); break;
case PropertyId::InterfaceName: g_value_set_string(value, self->config.network.interface_name); break;
case PropertyId::DestinationAddress: g_value_set_string(value, self->config.network.destination_address); break;
case PropertyId::DestinationPort: g_value_set_string(value, self->config.network.destination_port); break;
case PropertyId::DestinationPort: g_value_set_int(value, self->config.network.destination_port); break;

default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); break;
}
Expand All @@ -187,30 +192,76 @@ static gboolean gst_nmossender_sink_event(GstPad* pad, GstObject* parent, GstEve
gchar* caps_str = gst_caps_to_string(caps);
g_print("\n\nReceived CAPS from upstream: %s\n", caps_str);

for(guint i = 0; i < gst_caps_get_size(caps); i++)
const GstStructure* structure = gst_caps_get_structure(caps, 0);
const gchar* media_type = gst_structure_get_name(structure);

if(g_strcmp0(media_type, "audio/x-raw") == 0)
{
const GstStructure* structure = gst_caps_get_structure(caps, i);
// Convert the structure to a string
gchar* structure_str = gst_structure_to_string(structure);
GST_INFO_OBJECT(self, "Caps Structure %u: %s", i, structure_str);
g_free(structure_str);

// FIXME: Need to retrieve other fields
gint width, height;
const gchar* format;
if(gst_structure_get_int(structure, "width", &width))
{
self->config.media.width = width;
}
if(gst_structure_get_int(structure, "height", &height))
// DEBUG
g_print("Audio caps detected\n");
GST_INFO_OBJECT(self, "Audio caps detected");
//
gst_element_link(self->queue, self->audio_payloader);
gst_element_link(self->audio_payloader, self->udpsink);
self->caps = gst_caps_ref(caps);
self->config.is_audio = TRUE;
for(guint i = 0; i < gst_caps_get_size(caps); i++)
{
self->config.media.height = height;
gint rate, channels;
const gchar* format;
if(gst_structure_get_int(structure, "rate", &rate))
{
self->config.audio_sender_fields.sampling_rate = rate;
}
if(gst_structure_get_int(structure, "channels", &channels))
{
self->config.audio_sender_fields.number_of_channels = channels;
}
if((format = gst_structure_get_string(structure, "format")))
{
g_free(self->config.audio_sender_fields.format);
self->config.audio_sender_fields.format = g_strdup(translate_audio_format(format).c_str());
}
}
if((format = gst_structure_get_string(structure, "format")))
// DEBUG
g_print("Rate: %d, Channels: %d, Format: %s\n", self->config.audio_sender_fields.sampling_rate,
self->config.audio_sender_fields.number_of_channels, self->config.audio_sender_fields.format);
}
else if(g_strcmp0(media_type, "video/x-raw") == 0)
{
// DEBUG
g_print("Video caps detected\n");
GST_INFO_OBJECT(self, "Video caps detected");
//
gst_element_link(self->queue, self->video_payloader);
gst_element_link(self->video_payloader, self->udpsink);
self->caps = gst_caps_ref(caps);
self->config.is_audio = FALSE;
for(guint i = 0; i < gst_caps_get_size(caps); i++)
{
g_free(self->config.media.sampling);
self->config.media.sampling = g_strdup(format);
gint width, height;
const gchar* format;
if(gst_structure_get_int(structure, "width", &width))
{
self->config.video_media_fields.width = width;
}
if(gst_structure_get_int(structure, "height", &height))
{
self->config.video_media_fields.height = height;
}
if((format = gst_structure_get_string(structure, "format")))
{
g_free(self->config.video_media_fields.sampling);
self->config.video_media_fields.sampling = g_strdup(translate_video_format(format).c_str());
}
}

g_print("Width: %d, Height: %d, Format: %s\n", self->config.video_media_fields.width,
self->config.video_media_fields.height, self->config.video_media_fields.sampling);
}
else
{
GST_WARNING_OBJECT(self, "Unsupported media type: %s", media_type);
}

g_free(caps_str);
Expand All @@ -219,13 +270,11 @@ static gboolean gst_nmossender_sink_event(GstPad* pad, GstObject* parent, GstEve
{
GST_WARNING_OBJECT(self, "No caps found in CAPS event");
}

break;
}
default: break;
}

g_print("Width: %d, Height: %d, Format: %s\n", self->config.media.width, self->config.media.height,
self->config.media.sampling);
// Pass the event downstream
return gst_pad_event_default(pad, parent, event);
}
Expand All @@ -243,11 +292,20 @@ static GstStateChangeReturn gst_nmossender_change_state(GstElement* element, Gst

switch(transition)
{
case GST_STATE_CHANGE_NULL_TO_READY: {
case GST_STATE_CHANGE_PAUSED_TO_PLAYING: {

const auto node_config_json = create_node_config(self->config);
const auto device_config_json = create_device_config(self->config);
const auto sender_config_json = create_sender_config(self->config);
const auto node_config_json = create_node_config(self->config);
const auto device_config_json = create_device_config(self->config);
nlohmann::json_abi_v3_11_3::json sender_config_json = NULL;

if(self->config.is_audio)
{
sender_config_json = create_audio_sender_config(self->config);
}
else
{
sender_config_json = create_video_sender_config(self->config);
}

g_print("Node Configuration: %s\n", node_config_json.dump().c_str());
g_print("Device Configuration: %s\n", device_config_json.dump().c_str());
Expand Down Expand Up @@ -330,31 +388,33 @@ static void gst_nmossender_class_init(GstNmossenderClass* klass)
/* Object initialization */
static void gst_nmossender_init(GstNmossender* self)
{
self->payloader = gst_element_factory_make("rtpvrawpay", "payloader");
self->udpsink = gst_element_factory_make("udpsink", "udpsink");
self->queue = gst_element_factory_make("queue", "queue");
self->video_payloader = gst_element_factory_make("rtpvrawpay", "video_payloader");
self->audio_payloader = gst_element_factory_make("rtpL24pay", "audio_payloader");
self->udpsink = gst_element_factory_make("udpsink", "udpsink");

if(!self->payloader || !self->udpsink)
if(!self->queue || !self->video_payloader || !self->audio_payloader || !self->udpsink)
{
GST_ERROR_OBJECT(self, "Failed to create internal elements: payloader or udpsink");
GST_ERROR_OBJECT(self,
"Failed to create internal elements: queue or video_payloader or audio_playloader or udpsink");
return;
}

// default udpsink properties
// set properties
g_object_set(G_OBJECT(self->queue), "max-size-buffers", 1, NULL);
g_object_set(G_OBJECT(self->udpsink), "host", "127.0.0.1", "port", 9999, NULL);
self->config = create_default_config_fields();

// Add elements to the bin
gst_bin_add_many(GST_BIN(self), self->payloader, self->udpsink, NULL);
gst_element_link(self->payloader, self->udpsink);
gst_bin_add_many(GST_BIN(self), self->queue, self->video_payloader, self->audio_payloader, self->udpsink, NULL);

// create and configure the sink pad
GstPad* payloader_sinkpad = gst_element_get_static_pad(self->payloader, "sink");
GstPad* sink_ghost_pad = gst_ghost_pad_new("sink", payloader_sinkpad);
GstPad* queue_sinkpad = gst_element_get_static_pad(self->queue, "sink");
GstPad* sink_ghost_pad = gst_ghost_pad_new("sink", queue_sinkpad);

if(!sink_ghost_pad)
{
GST_ERROR_OBJECT(self, "Failed to create ghost pad for sink");
gst_object_unref(payloader_sinkpad);
gst_object_unref(queue_sinkpad);
return;
}

Expand All @@ -364,7 +424,7 @@ static void gst_nmossender_init(GstNmossender* self)

gst_element_add_pad(GST_ELEMENT(self), sink_ghost_pad);

gst_object_unref(payloader_sinkpad);
gst_object_unref(queue_sinkpad);
}

static gboolean plugin_init(GstPlugin* plugin)
Expand Down
92 changes: 74 additions & 18 deletions cpp/libs/gst_nmos_sender_plugin/utils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,24 +23,30 @@ config_fields_t create_default_config_fields()
config.device.label = g_strdup("OSSRF Device2");
config.device.description = g_strdup("OSSRF Device2");

// Initialize media_fields_t
config.media.width = 640;
config.media.height = 480;
config.media.frame_rate_num = 50;
config.media.frame_rate_density = 1;
config.media.sampling = g_strdup("YCbCr-4:2:2");
config.media.structure = g_strdup("progressive");
// Initialize video_media_fields_t
config.video_media_fields.width = 640;
config.video_media_fields.height = 480;
config.video_media_fields.frame_rate_num = 50;
config.video_media_fields.frame_rate_density = 1;
config.video_media_fields.sampling = g_strdup("YCbCr-4:2:2");
config.video_media_fields.structure = g_strdup("progressive");

// Initialize audio_media_fields_t
config.audio_sender_fields.format = g_strdup("audio/L24");
config.audio_sender_fields.number_of_channels = 1;
config.audio_sender_fields.packet_time = 1.000;
config.audio_sender_fields.sampling_rate = 48000;

// Initialize network_fields_t
config.network.source_address = g_strdup("192.168.1.120");
config.network.interface_name = g_strdup("wlp1s0");
config.network.destination_address = g_strdup("192.168.1.120");
config.network.destination_port = g_strdup("5004");
config.network.destination_port = 5004;

// Initialize config_fields_t
config.sender_id = g_strdup("e543a2c1-d6a2-47f5-8d14-296bb6714ef2");
config.sender_label = g_strdup("BISECT OSSRF sender video");
config.sender_description = g_strdup("BISECT OSSRF sender video");
config.sender_id = g_strdup("1c920570-e0b4-4637-b02c-26c9d4275c71");
config.sender_label = g_strdup("BISECT OSSRF Media Sender");
config.sender_description = g_strdup("BISECT OSSRF Media Sender");

return config;
}
Expand All @@ -65,7 +71,31 @@ json create_device_config(config_fields_t& config)
return device;
}

json create_sender_config(config_fields_t& config)
json create_video_sender_config(config_fields_t& config)
{
json sender = {
{"id", config.sender_id},
{"label", config.sender_label},
{"description", config.sender_description},
{"network",
{{"primary",
{{"source_address", config.network.source_address},
{"interface_name", config.network.interface_name},
{"destination_address", config.network.destination_address},
{"destination_port", config.network.destination_port}}}}},
{"payload_type", 97},
{"media_type", "video/raw"},
{"media",
{{"width", config.video_media_fields.width},
{"height", config.video_media_fields.height},
{"frame_rate",
{{"num", config.video_media_fields.frame_rate_num}, {"den", config.video_media_fields.frame_rate_density}}},
{"sampling", config.video_media_fields.sampling},
{"structure", config.video_media_fields.structure}}}};
return sender;
}

json create_audio_sender_config(config_fields_t& config)
{
json sender = {{"id", config.sender_id},
{"label", config.sender_label},
Expand All @@ -77,16 +107,42 @@ json create_sender_config(config_fields_t& config)
{"destination_address", config.network.destination_address},
{"destination_port", config.network.destination_port}}}}},
{"payload_type", 97},
{"media_type", "video/raw"},
{"media_type", "audio/L24"},
{"media",
{{"width", config.media.width},
{"height", config.media.height},
{"frame_rate", {{"num", config.media.frame_rate_num}, {"den", config.media.frame_rate_density}}},
{"sampling", config.media.sampling},
{"structure", config.media.structure}}}};
{{"number_of_channels", config.audio_sender_fields.number_of_channels},
{"sampling_rate", config.audio_sender_fields.sampling_rate},
{"packet_time", config.audio_sender_fields.packet_time}}}};
return sender;
}

std::string translate_video_format(const std::string& gst_format)
{
static const std::unordered_map<std::string, std::string> video_format_map = {
{"UYVP", "YCbCr-4:2:2"}, {"I420", "YCbCr-4:2:0"}, {"NV12", "YCbCr-4:2:0"}, {"RGB", "RGB-8:8:8"},
{"RGBA", "RGBA-8:8:8:8"}, {"BGR", "BGR-8:8:8"}, {"BGRA", "BGRA-8:8:8:8"}};

auto it = video_format_map.find(gst_format);
if(it != video_format_map.end())
{
return it->second;
}
return gst_format;
}

std::string translate_audio_format(const std::string& gst_format)
{
static const std::unordered_map<std::string, std::string> audio_format_map = {
{"S24BE", "L24"}, {"S16LE", "L16"}, {"F32LE", "F32"}, {"F64LE", "F64"},
{"S32LE", "L32"}, {"S24_32LE", "L24_32"}, {"U8", "U8"}};

auto it = audio_format_map.find(gst_format);
if(it != audio_format_map.end())
{
return it->second;
}
return gst_format;
}

expected<json> load_configuration_from_file(std::string_view config_file)
{
std::ifstream ifs(config_file.data());
Expand Down
Loading

0 comments on commit 8a6cf99

Please sign in to comment.