Skip to content

Commit

Permalink
Fix UploadLarge breaking large files on upload
Browse files Browse the repository at this point in the history
  • Loading branch information
@const-cloudinary
const-cloudinary committed Mar 14, 2024
1 parent defa08d commit c4d777b
Show file tree
Hide file tree
Showing 3 changed files with 49 additions and 39 deletions.
13 changes: 13 additions & 0 deletions CloudinaryDotNet.IntegrationTests/IntegrationTestBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Linq;
using System.Net.Http;
using System.Reflection;
using System.Security.Cryptography;
using System.Threading.Tasks;
using CloudinaryDotNet.Actions;
using NUnit.Framework;
Expand Down Expand Up @@ -507,6 +508,18 @@ protected string GetUniqueMetadataFieldLabel(string suffix = "")
return label;
}

protected static string GetFileMd5Sum(string filename)
{
using (var md5 = MD5.Create())
{
using (var stream = File.OpenRead(filename))
{
var hash = md5.ComputeHash(stream);
return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
}
}
}

[OneTimeTearDown]
public virtual void Cleanup()
{
Expand Down
68 changes: 31 additions & 37 deletions CloudinaryDotNet.IntegrationTests/UploadApi/UploadMethodsTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ public class UploadMethodsTest : IntegrationTestBase

protected string m_implicitTransformationText;

protected string m_largeFileEtag;
protected int m_largeFileLength;

public override void Initialize()
{
base.Initialize();
Expand All @@ -52,6 +55,9 @@ public override void Initialize()
m_transformationAr25, m_transformationAr69,
m_transformationAr30, m_transformationAr12,
m_eagerTransformation);

m_largeFileEtag = GetFileMd5Sum(m_testLargeImagePath);
m_largeFileLength = (int)new FileInfo(m_testLargeImagePath).Length;
}

[Test, RetryWithDelay]
Expand Down Expand Up @@ -576,7 +582,7 @@ public void TestUploadLargeNonSeekableStream()

var result = m_cloudinary.UploadLarge(uploadParams, TEST_CHUNK_SIZE);

AssertUploadLarge(result, bytes.Length);
AssertUploadLarge(result);
}
}

Expand All @@ -585,27 +591,25 @@ public void TestUploadLargeRawFiles()
{
// support uploading large raw files
var largeFilePath = m_testLargeImagePath;
int largeFileLength = (int)new FileInfo(largeFilePath).Length;

var uploadParams = GetUploadLargeRawParams(largeFilePath);

var result = m_cloudinary.UploadLarge(uploadParams, TEST_CHUNK_SIZE);

AssertUploadLarge(result, largeFileLength);
AssertUploadLarge(result);
}

[Test, RetryWithDelay]
public async Task TestUploadLargeRawFilesAsyncInParallel()
{
// support asynchronous uploading large raw files
var largeFilePath = m_testLargeImagePath;
int largeFileLength = (int)new FileInfo(largeFilePath).Length;

var uploadParams = GetUploadLargeRawParams(largeFilePath);

var result = await m_cloudinary.UploadLargeAsync<RawUploadResult>(uploadParams, TEST_CHUNK_SIZE, 2);

AssertUploadLarge(result, largeFileLength);
AssertUploadLarge(result);
}

private RawUploadParams GetUploadLargeRawParams(string path)
Expand All @@ -617,54 +621,53 @@ private RawUploadParams GetUploadLargeRawParams(string path)
};
}

private void AssertUploadLarge(RawUploadResult result, int fileLength)
private void AssertUploadLarge(RawUploadResult result, int fileLength = 0, string etag = null)
{
if (fileLength == 0)
{
fileLength = m_largeFileLength;
}
if (etag == null)
{
etag = m_largeFileEtag;
}
Assert.NotNull(result);
Assert.AreEqual(fileLength, result.Bytes, result.Error?.Message);
Assert.AreEqual(etag, result.Etag, result.Error?.Message);
}

[Test, RetryWithDelay]
public void TestUploadLarge()
{
// support uploading large image

var largeFilePath = m_testLargeImagePath;
int fileLength = (int)new FileInfo(largeFilePath).Length;
var result = m_cloudinary.UploadLarge(new ImageUploadParams()
{
File = new FileDescription(largeFilePath),
File = new FileDescription(m_testLargeImagePath),
Tags = m_apiTag
}, TEST_CHUNK_SIZE);

Assert.AreEqual(fileLength, result.Bytes, result.Error?.Message);
AssertUploadLarge(result);
}

[Test, RetryWithDelay]
public async Task TestUploadLargeAutoFilesAsync()
{
// support asynchronous uploading large raw files
var largeFilePath = m_testLargeImagePath;
int largeFileLength = (int)new FileInfo(largeFilePath).Length;

var uploadParams = new AutoUploadParams()
{
File = new FileDescription(largeFilePath),
File = new FileDescription(m_testLargeImagePath),
Tags = m_apiTag
};

var result = await m_cloudinary.UploadLargeAsync(uploadParams, TEST_CHUNK_SIZE);

AssertUploadLarge(result, largeFileLength);
AssertUploadLarge(result);

Assert.AreEqual("image", result.ResourceType);
}

[Test, RetryWithDelay]
public void TestUploadChunkSingleStream()
{
var largeFilePath = m_testLargeImagePath;
var largeFileLength = (int)new FileInfo(largeFilePath).Length;

ImageUploadResult result = null;

using (var currChunk = new MemoryStream())
Expand All @@ -677,7 +680,7 @@ public void TestUploadChunkSingleStream()

var buffer = new byte[TEST_CHUNK_SIZE];

using (var source = File.Open(largeFilePath, FileMode.Open))
using (var source = File.Open(m_testLargeImagePath, FileMode.Open))
{
int read;
while ((read = source.Read(buffer, 0, buffer.Length)) > 0)
Expand All @@ -693,16 +696,13 @@ public void TestUploadChunkSingleStream()
}
}

AssertUploadLarge(result, largeFileLength);
AssertUploadLarge(result);
Assert.AreEqual("image", result?.ResourceType);
}

[Test, RetryWithDelay]
public async Task TestUploadChunkMultipleStreamsCustomOffsetAsync()
{
var largeFilePath = m_testLargeImagePath;
var largeFileLength = (int)new FileInfo(largeFilePath).Length;

ImageUploadResult result = null;

var uploadParams = new ImageUploadParams()
Expand All @@ -714,7 +714,7 @@ public async Task TestUploadChunkMultipleStreamsCustomOffsetAsync()

var buffer = new byte[TEST_CHUNK_SIZE];

using (var source = File.Open(largeFilePath, FileMode.Open))
using (var source = File.Open(m_testLargeImagePath, FileMode.Open))
{
int read;
while ((read = source.Read(buffer, 0, buffer.Length)) > 0)
Expand All @@ -727,19 +727,16 @@ public async Task TestUploadChunkMultipleStreamsCustomOffsetAsync()
}
}

AssertUploadLarge(result, largeFileLength);
AssertUploadLarge(result);
Assert.AreEqual("image", result?.ResourceType);
}

[Test, RetryWithDelay]
public void TestUploadChunkMultipleFileParts()
{
var largeFilePath = m_testLargeImagePath;
var largeFileLength = (int)new FileInfo(largeFilePath).Length;

ImageUploadResult result = null;

var fileChunks = SplitFile(largeFilePath, TEST_CHUNK_SIZE, "multiple");
var fileChunks = SplitFile(m_testLargeImagePath, TEST_CHUNK_SIZE, "multiple");

var uploadParams = new ImageUploadParams()
{
Expand Down Expand Up @@ -773,17 +770,14 @@ public void TestUploadChunkMultipleFileParts()
}
}

AssertUploadLarge(result, largeFileLength);
AssertUploadLarge(result);
Assert.AreEqual("image", result?.ResourceType);
}

[Test, RetryWithDelay]
public void TestUploadChunkMultipleFilePartsInParallel()
{
var largeFilePath = m_testLargeImagePath;
var largeFileLength = (int)new FileInfo(largeFilePath).Length;

var fileChunks = SplitFile(largeFilePath, TEST_CHUNK_SIZE, "multiple_parallel");
var fileChunks = SplitFile(m_testLargeImagePath, TEST_CHUNK_SIZE, "multiple_parallel");

var uploadParams = new RawUploadParams()
{
Expand Down Expand Up @@ -820,7 +814,7 @@ public void TestUploadChunkMultipleFilePartsInParallel()

var uploadResult = resultCollection.FirstOrDefault(r => r.AssetId != null);

AssertUploadLarge(uploadResult, largeFileLength);
AssertUploadLarge(uploadResult);
Assert.AreEqual("raw", uploadResult?.ResourceType);
}

Expand Down
7 changes: 5 additions & 2 deletions CloudinaryDotNet/Core/LimitedStream.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ internal class LimitedStream : Stream
private readonly Stream originalStream;
private long remainingBytes;
private long startOffset;
private long currOffset;

/// <summary>
/// Initializes a new instance of the <see cref="LimitedStream"/> class.
Expand All @@ -24,7 +25,7 @@ public LimitedStream(Stream stream, long offset, long maxBytes)
{
originalStream = stream ?? throw new ArgumentNullException(nameof(stream));
remainingBytes = maxBytes;
startOffset = offset;
startOffset = currOffset = offset;

if (!stream.CanSeek)
{
Expand Down Expand Up @@ -64,9 +65,11 @@ public override long Position
public override int Read(byte[] buffer, int offset, int count)
{
// make sure stream is not moved around.
originalStream.Seek(startOffset, SeekOrigin.Begin);
originalStream.Seek(currOffset, SeekOrigin.Begin);

var bytesRead = originalStream.Read(buffer, offset, (int)Math.Min(count, remainingBytes));

currOffset += bytesRead;
remainingBytes -= bytesRead;

return bytesRead;
Expand Down

0 comments on commit c4d777b

Please sign in to comment.