Skip to content

Commit dfca00e

Browse files
SanyyazzzTRybina132
Sanyyazzz
and
TRybina132
authored
Upload chunks (#69)
* add integration tests project * add client * add test controller and controller tests * fix * add IStorageClient * fix storage client * accept file and content names in client method * add content name property * return status code when request is failed * add test for file upload from file info * move tests * add tests for upload methods with azure * fix tests * return downloaded file as stream * return localFile from download method * return result on download * add base test classes for controller tests * add methods upload in chunk --------- Co-authored-by: TRybina132 <rybina@managed-code.com>
1 parent 399a7fc commit dfca00e

20 files changed

Lines changed: 900 additions & 17 deletions

‎ManagedCode.Storage.Client/Class1.cs‎

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎ManagedCode.Storage.Client/IStorageClient.cs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
using System.IO;
2+
using System.Threading;
3+
using System.Threading.Tasks;
4+
using ManagedCode.Communication;
5+
using ManagedCode.Storage.Core.Models;
6+
7+
namespace ManagedCode.Storage.Client;
8+
9+
public interface IStorageClient
10+
{
11+
Task<Result<BlobMetadata>> UploadFile(Stream stream, string apiUrl, string contentName, CancellationToken cancellationToken = default);
12+
Task<Result<BlobMetadata>> UploadFile(FileInfo fileInfo, string apiUrl, string contentName, CancellationToken cancellationToken = default);
13+
Task<Result<BlobMetadata>> UploadFile(byte[] bytes, string apiUrl, string contentName, CancellationToken cancellationToken = default);
14+
Task<Result<BlobMetadata>> UploadFile(string base64, string apiUrl, string contentName, CancellationToken cancellationToken = default);
15+
Task<Result> UploadFileInChunks(Stream file, string apiUrl, int chunkSize, CancellationToken cancellationToken = default);
16+
Task<Result<LocalFile>> DownloadFile(string fileName, string apiUrl, string? path = null, CancellationToken cancellationToken = default);
17+
}

‎ManagedCode.Storage.Client/StorageClient.cs‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
using System;
2+
using System.IO;
3+
using System.Net;
4+
using System.Net.Http;
5+
using System.Net.Http.Json;
6+
using System.Net.Http.Headers;
7+
using System.Net.Http.Json;
8+
using System.Net.Http.Json;
9+
using System.Text.Json;
10+
using System.Threading;
11+
using System.Threading.Tasks;
12+
using ManagedCode.Communication;
13+
using ManagedCode.Storage.Core;
14+
using ManagedCode.Storage.Core.Models;
15+
16+
namespace ManagedCode.Storage.Client;
17+
18+
public class StorageClient : IStorageClient
19+
{
20+
private readonly HttpClient _httpClient;
21+
22+
public StorageClient(HttpClient httpClient)
23+
{
24+
_httpClient = httpClient;
25+
}
26+
27+
public async Task<Result<BlobMetadata>> UploadFile(Stream stream, string apiUrl, string contentName, CancellationToken cancellationToken = default)
28+
{
29+
var streamContent = new StreamContent(stream);
30+
31+
using (var formData = new MultipartFormDataContent())
32+
{
33+
formData.Add(streamContent, contentName, contentName);
34+
35+
var response = await _httpClient.PostAsync(apiUrl, formData, cancellationToken);
36+
37+
if (response.IsSuccessStatusCode)
38+
{
39+
var result = await response.Content.ReadFromJsonAsync<Result<BlobMetadata>>(cancellationToken: cancellationToken);
40+
return result;
41+
}
42+
43+
string content = await response.Content.ReadAsStringAsync(cancellationToken: cancellationToken);
44+
45+
return Result<BlobMetadata>.Fail(response.StatusCode, content);
46+
}
47+
}
48+
49+
public async Task<Result<BlobMetadata>> UploadFile(FileInfo fileInfo, string apiUrl, string contentName, CancellationToken cancellationToken = default)
50+
{
51+
using var streamContent = new StreamContent(fileInfo.OpenRead());
52+
53+
using (var formData = new MultipartFormDataContent())
54+
{
55+
formData.Add(streamContent, contentName, contentName);
56+
57+
var response = await _httpClient.PostAsync(apiUrl, formData, cancellationToken);
58+
59+
if (response.IsSuccessStatusCode)
60+
{
61+
var result = await response.Content.ReadFromJsonAsync<Result<BlobMetadata>>(cancellationToken: cancellationToken);
62+
return result;
63+
}
64+
65+
return Result<BlobMetadata>.Fail(response.StatusCode);
66+
}
67+
}
68+
69+
public async Task<Result<BlobMetadata>> UploadFile(byte[] bytes, string apiUrl, string contentName, CancellationToken cancellationToken = default)
70+
{
71+
using (var stream = new MemoryStream())
72+
{
73+
stream.Write(bytes, 0, bytes.Length);
74+
75+
using var streamContent = new StreamContent(stream);
76+
77+
using (var formData = new MultipartFormDataContent())
78+
{
79+
formData.Add(streamContent, contentName, contentName);
80+
81+
var response = await _httpClient.PostAsync(apiUrl, formData, cancellationToken);
82+
83+
if (response.IsSuccessStatusCode)
84+
{
85+
var result = await response.Content.ReadFromJsonAsync<Result<BlobMetadata>>(cancellationToken: cancellationToken);
86+
return result;
87+
}
88+
89+
return Result<BlobMetadata>.Fail(response.StatusCode);
90+
}
91+
}
92+
}
93+
94+
public async Task<Result<BlobMetadata>> UploadFile(string base64, string apiUrl, string contentName, CancellationToken cancellationToken = default)
95+
{
96+
byte[] fileAsBytes = Convert.FromBase64String(base64);
97+
using var fileContent = new ByteArrayContent(fileAsBytes);
98+
99+
using var formData = new MultipartFormDataContent();
100+
101+
formData.Add(fileContent, contentName, contentName);
102+
103+
var response = await _httpClient.PostAsync(apiUrl, formData, cancellationToken);
104+
105+
if (response.IsSuccessStatusCode)
106+
{
107+
return await response.Content.ReadFromJsonAsync<Result<BlobMetadata>>(cancellationToken: cancellationToken);
108+
}
109+
110+
return Result<BlobMetadata>.Fail(response.StatusCode);
111+
}
112+
113+
public async Task<Result<LocalFile>> DownloadFile(string fileName, string apiUrl, string? path = null, CancellationToken cancellationToken = default)
114+
{
115+
try
116+
{
117+
var response = await _httpClient.GetStreamAsync($"{apiUrl}/{fileName}", cancellationToken);
118+
119+
var localFile = path is null ? await LocalFile.FromStreamAsync(response, fileName) : await LocalFile.FromStreamAsync(response, path, fileName);
120+
121+
return Result<LocalFile>.Succeed(localFile);
122+
}
123+
catch (HttpRequestException e)
124+
{
125+
return Result<LocalFile>.Fail(e.StatusCode ?? HttpStatusCode.InternalServerError);
126+
}
127+
}
128+
129+
public async Task<Result> UploadFileInChunks(Stream file, string apiUrl, int chunkSize, CancellationToken cancellationToken)
130+
{
131+
var buffer = new byte[chunkSize];
132+
int bytesRead;
133+
int chunkIndex = 0;
134+
135+
while ((bytesRead = await file.ReadAsync(buffer, 0, buffer.Length, cancellationToken)) > 0)
136+
{
137+
// Create a MemoryStream for the current chunk.
138+
using (var memoryStream = new MemoryStream(buffer, 0, bytesRead))
139+
{
140+
var content = new StreamContent(memoryStream);
141+
142+
using (var chunk = new MultipartFormDataContent())
143+
{
144+
chunk.Add(content, "file", "file");
145+
146+
var byteArrayContent = new ByteArrayContent(await chunk.ReadAsByteArrayAsync(cancellationToken));
147+
// Send the current chunk to the API endpoint.
148+
var response = await _httpClient.PostAsync(apiUrl, byteArrayContent, cancellationToken);
149+
150+
if (!response.IsSuccessStatusCode)
151+
{
152+
return Result.Fail();
153+
}
154+
}
155+
}
156+
}
157+
158+
var mergeResult = await _httpClient.PostAsync(apiUrl + "/complete", JsonContent.Create("file"));
159+
160+
return await mergeResult.Content.ReadFromJsonAsync<Result>();
161+
}
162+
}

‎ManagedCode.Storage.Core/Models/LocalFile.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,17 @@ public static async Task<LocalFile> FromStreamAsync(Stream stream)
150150
await file.FileStream.DisposeAsync();
151151
return file;
152152
}
153+
154+
public static async Task<LocalFile> FromStreamAsync(Stream stream, string path, string fileName)
155+
{
156+
var pathWithName = Path.Combine(path, $"{fileName}.tmp");
157+
var file = new LocalFile(pathWithName);
158+
159+
await stream.CopyToAsync(file.FileStream);
160+
await file.FileStream.DisposeAsync();
161+
162+
return file;
163+
}
153164

154165
public static async Task<LocalFile> FromStreamAsync(Stream stream, string fileName)
155166
{

‎ManagedCode.Storage.IntegrationTests/Constants/ApiEndpoints.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
namespace ManagedCode.Storage.IntegrationTests.Constants;
2+
3+
public static class ApiEndpoints
4+
{
5+
public const string Azure = "azure";
6+
7+
public static class Base
8+
{
9+
public const string UploadFile = "{0}/upload";
10+
public const string UploadFileChunks = "{0}/upload-chunks";
11+
public const string DownloadFile = "{0}/download";
12+
}
13+
}

‎ManagedCode.Storage.IntegrationTests/Helpers/Crc32Helper.cs‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
namespace ManagedCode.Storage.IntegrationTests.Helpers;
2+
3+
public static class Crc32Helper
4+
{
5+
private static readonly uint[] Crc32Table;
6+
private const uint polynomial = 0xedb88320;
7+
8+
static Crc32Helper()
9+
{
10+
Crc32Table = new uint[256];
11+
12+
for (int i = 0; i < 256; i++)
13+
{
14+
uint crc = (uint)i;
15+
for (int j = 8; j > 0; j--)
16+
{
17+
if ((crc & 1) == 1)
18+
crc = (crc >> 1) ^ polynomial;
19+
else
20+
crc >>= 1;
21+
}
22+
Crc32Table[i] = crc;
23+
}
24+
}
25+
26+
public static uint Calculate(byte[] bytes)
27+
{
28+
uint crcValue = 0xffffffff;
29+
30+
foreach (byte by in bytes)
31+
{
32+
byte tableIndex = (byte)(((crcValue) & 0xff) ^ by);
33+
crcValue = Crc32Table[tableIndex] ^ (crcValue >> 8);
34+
}
35+
return ~crcValue;
36+
}
37+
}

‎ManagedCode.Storage.IntegrationTests/Helpers/FileHelper.cs‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
using ManagedCode.MimeTypes;
2+
using ManagedCode.Storage.Core.Models;
3+
using Microsoft.AspNetCore.Http;
4+
5+
namespace ManagedCode.Storage.IntegrationTests.Helpers;
6+
7+
public static class FileHelper
8+
{
9+
private static readonly Random Random = new();
10+
11+
public static LocalFile GenerateLocalFile(LocalFile file, int sizeInMegabytes)
12+
{
13+
var sizeInBytes = sizeInMegabytes * 1024 * 1024;
14+
15+
using (var fileStream = file.FileStream)
16+
{
17+
Random random = new Random();
18+
byte[] buffer = new byte[1024]; // Buffer for writing in chunks
19+
20+
while (sizeInBytes > 0)
21+
{
22+
int bytesToWrite = (int) Math.Min(sizeInBytes, buffer.Length);
23+
24+
for (int i = 0; i < bytesToWrite; i++)
25+
{
26+
buffer[i] = (byte) random.Next(65, 91); // 'A' to 'Z'
27+
if (random.Next(2) == 0)
28+
{
29+
buffer[i] = (byte) random.Next(97, 123); // 'a' to 'z'
30+
}
31+
}
32+
33+
fileStream.Write(buffer, 0, bytesToWrite);
34+
sizeInBytes -= bytesToWrite;
35+
}
36+
}
37+
38+
return file;
39+
}
40+
41+
/*public static IFormFile GenerateFormFile(string fileName, int byteSize)
42+
{
43+
var localFile = GenerateLocalFile(fileName, byteSize);
44+
45+
var ms = new MemoryStream();
46+
localFile.FileStream.CopyTo(ms);
47+
var formFile = new FormFile(ms, 0, ms.Length, fileName, fileName)
48+
{
49+
Headers = new HeaderDictionary(),
50+
ContentType = MimeHelper.GetMimeType(localFile.FileInfo.Extension)
51+
};
52+
53+
localFile.Dispose();
54+
55+
return formFile;
56+
}
57+
58+
public static string GenerateRandomFileName(string extension = "txt")
59+
{
60+
return $"{Guid.NewGuid().ToString("N").ToLowerInvariant()}.{extension}";
61+
}
62+
63+
public static string GenerateRandomFileContent()
64+
{
65+
const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ-0123456789_abcdefghijklmnopqrstuvwxyz";
66+
67+
return new string(Enumerable.Repeat(chars, 250_000)
68+
.Select(s => s[Random.Next(s.Length)])
69+
.ToArray());
70+
}*/
71+
}

‎ManagedCode.Storage.IntegrationTests/ManagedCode.Storage.IntegrationTests.csproj‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>net7.0</TargetFramework>
5+
<ImplicitUsings>enable</ImplicitUsings>
6+
<Nullable>enable</Nullable>
7+
<GenerateProgramFile>false</GenerateProgramFile>
8+
<IsPackable>false</IsPackable>
9+
</PropertyGroup>
10+
11+
<ItemGroup>
12+
<PackageReference Include="FluentAssertions" Version="6.12.0" />
13+
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="7.0.12" />
14+
<PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="7.0.12" />
15+
<PackageReference Include="Microsoft.AspNetCore.TestHost" Version="7.0.12" />
16+
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
17+
<PackageReference Include="Testcontainers.Azurite" Version="3.5.0" />
18+
<PackageReference Include="xunit" Version="2.4.2" />
19+
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
20+
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
21+
<PrivateAssets>all</PrivateAssets>
22+
</PackageReference>
23+
<PackageReference Include="coverlet.collector" Version="3.1.2">
24+
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
25+
<PrivateAssets>all</PrivateAssets>
26+
</PackageReference>
27+
</ItemGroup>
28+
29+
<ItemGroup>
30+
<ProjectReference Include="..\ManagedCode.Storage.Aws\ManagedCode.Storage.Aws.csproj" />
31+
<ProjectReference Include="..\ManagedCode.Storage.Azure\ManagedCode.Storage.Azure.csproj" />
32+
<ProjectReference Include="..\ManagedCode.Storage.Client\ManagedCode.Storage.Client.csproj" />
33+
<ProjectReference Include="..\ManagedCode.Storage.Core\ManagedCode.Storage.Core.csproj" />
34+
<ProjectReference Include="..\ManagedCode.Storage.FileSystem\ManagedCode.Storage.FileSystem.csproj" />
35+
<ProjectReference Include="..\ManagedCode.Storage.Google\ManagedCode.Storage.Google.csproj" />
36+
<ProjectReference Include="..\ManagedCode.Storage.Server\ManagedCode.Storage.Server.csproj" />
37+
</ItemGroup>
38+
39+
</Project>

0 commit comments

Comments
 (0)