Skip to content

Commit

Permalink
Expose Bitrate and Change Render Streaming (#6)
Browse files Browse the repository at this point in the history
- Added Bit Rate
- Added New VR Camera Streaming Component
- Updated Samples with new Prefab Changes
  • Loading branch information
vmohan7 authored Aug 18, 2021
1 parent 9ed56fd commit 4b1d068
Show file tree
Hide file tree
Showing 8 changed files with 192 additions and 170 deletions.
12 changes: 12 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,18 @@ All notable changes to com.unity.renderstreaming package will be documented in t
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).

## [0.4.0] - 2021-08-17

### Changed

- **MAJOR** Combined left eye and right eye into a single render texture on one video track to increase encoding performance & maintain synconization between eyes
- Removed Video Resizing as this implementation needs to be re-evaluated
- Increased default resolution to 2700 x 1500 (both eyes)

### Added

- Added new VRCamStreaming component that exposes bitrate and framerate encoder parameters

## [0.3.0] - 2021-08-10

### Changed
Expand Down
59 changes: 22 additions & 37 deletions Runtime/Prefabs/Render Streaming Services.prefab
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ GameObject:
- component: {fileID: 1793859182182230346}
- component: {fileID: 1793859182182230347}
- component: {fileID: 1270632014}
- component: {fileID: 1084652618}
m_Layer: 0
m_Name: WebVR Camera
m_TagString: Untagged
Expand Down Expand Up @@ -49,9 +50,7 @@ MonoBehaviour:
m_EditorClassIdentifier:
local: 0
label:
VRCameras:
- {fileID: 1793859182864016192}
- {fileID: 1793859183251251332}
VRCameras: {fileID: 1084652618}
VRPoseEvent:
m_PersistentCalls:
m_Calls:
Expand Down Expand Up @@ -106,6 +105,24 @@ MonoBehaviour:
VRAxisEvent:
m_PersistentCalls:
m_Calls: []
--- !u!114 &1084652618
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1793859182182230341}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 88dc1ef0a0e7d1b4a897f095b852ab38, type: 3}
m_Name:
m_EditorClassIdentifier:
streamingSize: {x: 2700, y: 1500}
leftEye: {fileID: 1793859182864016192}
rightEye: {fileID: 1793859183251251332}
depth: 0
antiAliasing: 1
BIT_RATE: 9000000
--- !u!1 &1793859182664022047
GameObject:
m_ObjectHideFlags: 0
Expand Down Expand Up @@ -256,7 +273,6 @@ GameObject:
m_Component:
- component: {fileID: 1793859182864016220}
- component: {fileID: 1793859182864016192}
- component: {fileID: 1793859182864016198}
m_Layer: 0
m_Name: Left Eye Render Streaming Camera
m_TagString: Untagged
Expand Down Expand Up @@ -321,21 +337,6 @@ Camera:
m_OcclusionCulling: 1
m_StereoConvergence: 10
m_StereoSeparation: 0.022
--- !u!114 &1793859182864016198
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1793859182864016223}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: e3379f8b7ab35724b965e26351c2004e, type: 3}
m_Name:
m_EditorClassIdentifier:
streamingSize: {x: 1440, y: 1600}
depth: 0
antiAliasing: 2
--- !u!1 &1793859182968099965
GameObject:
m_ObjectHideFlags: 0
Expand Down Expand Up @@ -529,7 +530,6 @@ GameObject:
m_Component:
- component: {fileID: 1793859183251251333}
- component: {fileID: 1793859183251251332}
- component: {fileID: 1793859183251251328}
m_Layer: 0
m_Name: Right Eye Render Streaming Camera
m_TagString: Untagged
Expand Down Expand Up @@ -594,21 +594,6 @@ Camera:
m_OcclusionCulling: 1
m_StereoConvergence: 10
m_StereoSeparation: 0.022
--- !u!114 &1793859183251251328
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1793859183251251330}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: e3379f8b7ab35724b965e26351c2004e, type: 3}
m_Name:
m_EditorClassIdentifier:
streamingSize: {x: 1440, y: 1600}
depth: 0
antiAliasing: 2
--- !u!1 &1793859183478774782
GameObject:
m_ObjectHideFlags: 0
Expand Down Expand Up @@ -730,9 +715,9 @@ MonoBehaviour:
m_Script: {fileID: 11500000, guid: 31b94e633688a294abc5410d3190b6f5, type: 3}
m_Name:
m_EditorClassIdentifier:
gameID:
streams:
- {fileID: 1793859182864016198}
- {fileID: 1793859183251251328}
- {fileID: 1084652618}
- {fileID: 1793859183478774780}
- {fileID: 1793859182182230347}
--- !u!1 &9010668948091610382
Expand Down
13 changes: 10 additions & 3 deletions Runtime/Scripts/VRBroadcast.cs
Original file line number Diff line number Diff line change
Expand Up @@ -114,9 +114,16 @@ public void OnAddChannel(SignalingEventData data) {
private string GetGameID(string sdp) {
string[] parameters = sdp.Split('\n'); //split on new line
string customParam = parameters[parameters.Length - 2].Split('=')[1]; //get line before last new line and split on =
string value = ""; //get dictionary value
JsonConvert.DeserializeObject<Dictionary<string, string>>(customParam).TryGetValue("user", out value);
return value;

Dictionary<string, string> jsonMap = JsonConvert.DeserializeObject<Dictionary<string, string>>(customParam);
if (jsonMap != null) {
string value = ""; //get dictionary value
jsonMap.TryGetValue("user", out value);
return value;
} else {
return ""; //TODO: check if this is a valid fail case
}

}
}
}
98 changes: 98 additions & 0 deletions Runtime/Scripts/VRCamStream.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
using UnityEngine;
using Unity.RenderStreaming;
using Unity.WebRTC;

namespace FusedVR.VRStreaming {
public class VRCamStream : VideoStreamBase {

#region Variables
[SerializeField]
[Tooltip("The Left Eye of the VR Camera")]
private Camera leftEye;

[SerializeField]
[Tooltip("The Right Eye of the VR Camera")]
private Camera rightEye;

[SerializeField]
[Tooltip("Defines the depth buffer used for render streaming (0, 16, 24, 32)")]
private int depth = 0;

[SerializeField]
[Tooltip("Defines the number of samples for anti-aliasing (1, 2, 4, 8)")]
private int antiAliasing = 1;

[SerializeField]
[Tooltip("Defines the default bitrate to be used by the encoders in bits per second")]
private ulong BIT_RATE = 9000000; //default bitrate of 9 Mbps

/// <summary>
/// The Main Connection ID that this instance is connected with
/// </summary>
private string mainConnection = "";


public const uint MAX_FRAMERATE = 90; //default max framerate target

public override Texture SendTexture => leftEye.targetTexture; //should be the same as right eye
#endregion

#region Events
// Start is called before the first frame update
void Start() {
OnStartedStream += StartStream;
}

private void StartStream(string connectionId) {
mainConnection = connectionId;
ChangeSendParameters(BIT_RATE, MAX_FRAMERATE);
}

/// <summary>
/// Assigns Render Textures to our cameras and then
/// Creates a Video Streaming Track that references the render textures
/// </summary>
protected override MediaStreamTrack CreateTrack() {

RenderTextureFormat format = WebRTC.GetSupportedRenderTextureFormat(SystemInfo.graphicsDeviceType);
RenderTexture rt = new RenderTexture(streamingSize.x, streamingSize.y, depth, format) {
antiAliasing = antiAliasing
};
rt.Create();

leftEye.targetTexture = rt;
leftEye.rect = new Rect(Vector2.zero, new Vector2(0.5f, 1f));

rightEye.targetTexture = rt;
rightEye.rect = new Rect(new Vector2(0.5f, 0f), new Vector2(0.5f, 1f));

return new VideoStreamTrack("VR Camera", rt);
}
#endregion

#region Public Methods
/// <summary>
/// Change Parameters associated with encoders for sending data to browser
/// </summary>
public void ChangeSendParameters(ulong? bitrate , uint? framerate) {
if (Senders.TryGetValue(mainConnection, out var sender)) {
RTCRtpSendParameters parameters = sender.GetParameters();
foreach (var encoding in parameters.encodings) {
if (bitrate != null) {
encoding.minBitrate = bitrate;
encoding.maxBitrate = bitrate;
}

if (framerate != null) {
encoding.maxFramerate = framerate;
}
}

sender.SetParameters(parameters);
}
}
#endregion

}
}

11 changes: 11 additions & 0 deletions Runtime/Scripts/VRCamStream.cs.meta

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

13 changes: 7 additions & 6 deletions Runtime/Scripts/VRInputManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ public class VRInputManager : InputChannelReceiverBase {
/// Camera that are used for VR Render Streaming
/// </summary>
[Tooltip("The Cameras that are responsible for VR Render Streaming")]
public Camera[] VRCameras;
public VRCamStream VRCameras;

#region Constants
/// <summary>
Expand Down Expand Up @@ -126,11 +126,12 @@ protected override void OnMessage(byte[] bytes) {
case VRDataType.Display:
int width = (int)BitConverter.ToUInt32(bytes, 2);
int height = (int)BitConverter.ToUInt32(bytes, 6);
foreach (Camera cam in VRCameras) {
cam.targetTexture.Release();
cam.targetTexture.width = width / 2; // half since the the width covers both eyes
cam.targetTexture.height = height;
}
//foreach (Camera cam in VRCameras) {
// cam.targetTexture.Release();
// cam.targetTexture.width = width;
// cam.targetTexture.height = height;
//}
//TODO: need to find proper way to resize texture based on data so that the video channel updates

break;
}
Expand Down
Loading

0 comments on commit 4b1d068

Please sign in to comment.