Intro
I wanted to send files to Web application (made by ASP.NET Core).If the file size had been small, I didn't need do any special things.
But when I tried to send a large file, the error was occurred by ASP.NET Core's limitation.
Though I could change the settings, but I didn't want to do that, because I hadn't known the file sizes what would been actually using.
So I splitted the data into chunks first, and sent them. After receiving all chunks, I merged them into one file.
There might be some libraries or APIs (ex. Stream API) what did them automatically, but I couldn't find them.
What I did
- [ASP.NET Core] Make CORS enabled
- [Angular] Split a large file into chunks
- [Angular][ASP.NET Core] Send and receive data as form data
- [ASP.NET Core] Merge chunks into one file
[ASP.NET Core] Make CORS enabled
Because the client side application(Angular) and the server side application(ASP.NET Core) had been separated, I had to make CORS(Cross-Origin Requests) enabled before accessing each other.Enable Cross-Origin Requests (CORS) in ASP.NET Core - Microsoft Docs
Startup.cs
using Microsoft.AspNetCore.Builder;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
...
namespace UpgradeSample
{
public class Startup
{
private IConfigurationRoot Configuration { get; }
private static readonly string AllowedOrigins = "_allowedOrigins";
...
public void ConfigureServices(IServiceCollection services)
{
...
services.AddCors(options =>
{
options.AddPolicy(AllowedOrigins,
builder =>
{
builder.WithOrigins("http://localhost:5000",
"http://localhost:4200")
.AllowAnyHeader()
.AllowAnyMethod();
});
});
services.AddControllers();
...
}
public void Configure(IApplicationBuilder app, IHostEnvironment env)
{
...
app.UseCors(AllowedOrigins);
app.UseStaticFiles();
app.UseRouting();
app.UseAuthentication();
app.UseAuthorization();
app.UseEndpoints(endpoints =>
{
endpoints.MapControllers();
});
}
}
}
[Angular] Split a large file into chunks
I used File API to get file, and split it into chunks and send them.file-upload.component.html
<input type="file" (change)="fileChanged($event.target)" >
<button (click)="saveFile()">Save</button>
file-upload.component.ts
import { Component, OnInit } from '@angular/core';
import {FileUploaderService} from './file-uploader.service';
@Component({
selector: 'app-file-upload',
templateUrl: './file-upload.component.html',
styleUrls: ['./file-upload.component.css']
})
export class FileUploadComponent implements OnInit {
private uploadFile: File;
constructor(private fileUploader: FileUploaderService) { }
ngOnInit() {
}
public fileChanged(target: EventTarget) {
const fileElement = target as HTMLInputElement;
if (fileElement == null ||
fileElement.files == null ||
fileElement.files.length <= 0) {
return;
}
// the first file is set as upload target.
this.uploadFile = fileElement.files[0];
}
public saveFile() {
if (this.uploadFile == null) {
return;
}
this.fileUploader.upload(this.uploadFile)
.subscribe(result => console.log(result), // onNext
error => console.erro(error), // onError
() => console.log('finished')); // onComplete
}
}
I splitted the file data by FileReader.
FileReader - Web APIs | MDN
FileReader.readAsArrayBuffer() - Web APIs | MDN
google chrome - Split an uploaded file into multiple chunks using javascript - Stack Overflow
Uint8Array - JavaScript | MDN
JavaScriptのStreams APIで細切れのデータを読み書きする
File - Web APIs | MDN
file-uploader.service.ts
import { Injectable } from '@angular/core';
import {HttpClient} from '@angular/common/http';
import {Observable, Observer} from 'rxjs';
import {UploadFileResult} from './upload-file-result';
import {UploadResult} from './upload-result';
@Injectable({
providedIn: 'root'
})
export class FileUploaderService {
constructor(private httpClient: HttpClient) { }
private chunkSize = 1048576;
private static getErrorResult(reason: any): UploadFileResult {
return {
result: {
succeeded: false,
errorMessage: reason
},
tmpDirectoryName: ''
};
}
public upload(file: File): Observable<boolean> {
return new Observable<boolean>(observer => {
const reader = new FileReader();
const fileName = file.name;
reader.onload = (e: ProgressEvent) =>
// after reading, start sending data
this.sendBufferData(reader.result, fileName, observer);
// read file data as ArrayBuffer
reader.readAsArrayBuffer(file);
});
}
private async sendBufferData(readResult: string|ArrayBuffer,
fileName: string,
observer: Observer<boolean>) {
if (readResult == null) {
observer.error('failed reading file data');
return;
}
// split ArrayBuffer by 1MB and send to server
}
[Angular][ASP.NET Core] Send and receive data as form data & Merge chunks into one file
Send data
file-uploader.service.ts
import { Injectable } from '@angular/core';
import {HttpClient} from '@angular/common/http';
import {Observable, Observer} from 'rxjs';
import {UploadFileResult} from './upload-file-result';
import {UploadResult} from './upload-result';
@Injectable({
providedIn: 'root'
})
export class FileUploaderService {
constructor(private httpClient: HttpClient) { }
private chunkSize = 1048576;
private static getErrorResult(reason: any): UploadFileResult {
return {
result: {
succeeded: false,
errorMessage: reason
},
tmpDirectoryName: ''
};
}
public upload(file: File): Observable<boolean> {
return new Observable<boolean>(observer => {
const reader = new FileReader();
const fileName = file.name;
reader.onload = (e: ProgressEvent) =>
this.sendBufferData(reader.result, fileName, observer);
reader.readAsArrayBuffer(file);
});
}
private async sendBufferData(readResult: string|ArrayBuffer,
fileName: string,
observer: Observer<boolean>) {
if (readResult == null) {
observer.error('failed reading file data');
return;
}
// create directory for saving chunks by server side application.
await this.startSendingData(fileName)
.then(result => this.sendChunks(
new Uint8Array(readResult as ArrayBuffer),
result.tmpDirectoryName
))
.then(result => this.endSendingData(
fileName,
result.tmpDirectoryName
))
.then(result => {
if (result.succeeded) {
observer.next(true);
observer.complete();
} else {
observer.error(result);
}
})
.catch(reason => observer.error(reason));
}
private startSendingData(fileName: string): Promise<UploadFileResult> {
return new Promise<UploadFileResult>(((resolve, reject) => {
const formData = new FormData();
formData.append('fileName', fileName);
this.httpClient.post<UploadFileResult>(
'http://localhost:5000/files/start',
formData, { })
.toPromise()
.then(result => {
if (result.result.succeeded) {
resolve(result);
} else {
reject(result);
}
})
.catch(reason => reject(FileUploaderService.getErrorResult(reason)));
}));
}
private async sendChunks(buffer: Uint8Array,
tmpDirectoryName: string): Promise<UploadFileResult> {
return new Promise<UploadFileResult>(async (resolve, reject) => {
let fileIndex = 0;
let fileIndex = 0;
const sendChunkPromises = new Array<Promise<UploadResult>>();
for (let i = 0; i < buffer.length; i += this.chunkSize) {
let indexTo = i + this.chunkSize;
if (indexTo >= buffer.length) {
indexTo = buffer.length - 1; // for last data.
}
const formData = new FormData();
formData.append('file', new Blob([buffer.subarray(i, indexTo)]));
const promise = this.httpClient.post<UploadResult>(
'http://localhost:5000/files/chunk',
formData,
{
headers: {
tmpDirectory: tmpDirectoryName,
index: fileIndex.toString()
}
})
.toPromise();
sendChunkPromises.push(promise);
fileIndex += 1;
}
await Promise.all(sendChunkPromises)
.then(results => {
if (results.some(r => r.succeeded === false)) {
reject('failed uploading');
} else {
resolve({
result: {
succeeded: true,
errorMessage: ''
},
tmpDirectoryName
});
}
})
.catch(reason => reject(FileUploaderService.getErrorResult(reason)));
});
}
private endSendingData(fileName: string, tmpDirectoryName: string): Promise<UploadResult> {
return new Promise<UploadResult>(((resolve, reject) => {
const formData = new FormData();
formData.append('fileName', fileName);
formData.append('tmpDirectory', tmpDirectoryName);
this.httpClient.post<UploadResult>(
'http://localhost:5000/files/end',
formData, { })
.toPromise()
.then(result => {
if (result.succeeded) {
resolve(result);
} else {
reject(result);
}
})
.catch(reason => reject(FileUploaderService.getErrorResult(reason).result));
}));
}
}
Receive form data
FileApiController.cs
using System;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Files;
namespace UpgradeSample.Controllers
{
public class FileApiController: Controller
{
private readonly ILocalFileAccessor _localFileAccessor;
public FileApiController(ILocalFileAccessor localFileAccessor)
{
_localFileAccessor = localFileAccessor;
}
[HttpPost]
[Route("/files/start")]
[Produces("application/json")]
public UploadFileResult StartUploading([FromForm] string fileName)
{
if (string.IsNullOrEmpty(fileName))
{
return new UploadFileResult
{
Result = GetFailedResult("invalid file name"),
};
}
string directoryName = _localFileAccessor.CreateTmpDirectory(fileName);
if (string.IsNullOrEmpty(directoryName))
{
return new UploadFileResult
{
Result = GetFailedResult("failed create directory"),
};
}
return new UploadFileResult
{
Result = GetSuccessResult(),
TmpDirectoryName = directoryName,
};
}
[HttpPost]
[Route("/files/chunk")]
[Produces("application/json")]
public async Task<UploadResult> UploadChunk([FromForm] IFormFile file)
{
string directoryName = Request.Headers["tmpDirectory"];
if (string.IsNullOrEmpty(directoryName))
{
return GetFailedResult("no directory name");
}
if (int.TryParse(Request.Headers["index"], out int fileIndex) == false)
{
return GetFailedResult("no file index");
}
if (file == null)
{
return GetFailedResult("no file data");
}
bool result = await _localFileAccessor.SaveChunkAsync(directoryName, fileIndex, file);
return (result)? GetSuccessResult(): GetFailedResult("failed saving chunk data");
}
[HttpPost]
[Route("/files/end")]
[Produces("application/json")]
public async Task<UploadResult> EndUploading([FromForm] string fileName, string tmpDirectory)
{
if (string.IsNullOrEmpty(fileName))
{
return GetFailedResult("invalid file name");
}
if (string.IsNullOrEmpty(tmpDirectory))
{
return GetFailedResult("invalid directory");
}
bool result = await _localFileAccessor.MergeChunksAsync(tmpDirectory, fileName);
return (result)? GetSuccessResult(): GetFailedResult("failed saving file");
}
private static UploadResult GetSuccessResult()
{
return new UploadResult
{
Succeeded = true,
};
}
private static UploadResult GetFailedResult(string reason)
{
return new UploadResult
{
Succeeded = false,
ErrorMessage = reason,
};
}
}
}
Save chunks and merge into one file
ILocalFileAccessor.cs
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
namespace Files
{
public interface ILocalFileAccessor
{
string CreateTmpDirectory(string fileName);
Task<bool> SaveChunkAsync(string directoryName, int fileIndex, IFormFile file);
Task<bool> MergeChunksAsync(string tmpDirectoryName, string fileName);
}
}
LocalFileAccessor.cs
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.FileProviders;
namespace Files
{
public class LocalFileAccessor: ILocalFileAccessor
{
private const string TmpFileDirectory = @"C:/Users/example/Documents/workspace/Files/tmp/";
private const string ActualFileDirectory = @"C:/Users/example/Documents/workspace/Files/";
// Create root directories
public LocalFileAccessor()
{
CreateRootDirectories();
}
public string CreateTmpDirectory(string fileName)
{
CreateRootDirectories();
// Create directory for saving chunks
string directoryName = GetUniqueDirectoryPath(TmpFileDirectory,
$"{DateTime.Now:yyyyMMddHHmmssfff}_{fileName}");
DirectoryInfo info = Directory.CreateDirectory($"{TmpFileDirectory}{directoryName}");
return directoryName;
}
public async Task<bool> SaveChunkAsync(string directoryName, int fileIndex, IFormFile file)
{
string directoryPath = $"{TmpFileDirectory}{directoryName}";
if (Directory.Exists(directoryPath) == false)
{
return false;
}
string filePath = $"{directoryPath}/{fileIndex.ToString()}_{directoryName}";
await using FileStream stream = new FileStream(filePath, FileMode.Create);
await file.CopyToAsync(stream);
return true;
}
public async Task<bool> MergeChunksAsync(string tmpDirectoryName, string fileName)
{
CreateRootDirectories();
string tmpDirectoryPath = $"{TmpFileDirectory}{tmpDirectoryName}";
if (Directory.Exists(tmpDirectoryPath) == false)
{
return false;
}
// Read all chunks by byte data
List<byte> readBytes = new List<byte>();
using(PhysicalFileProvider provider = new PhysicalFileProvider(tmpDirectoryPath))
{
foreach(IFileInfo fileInfo in provider.GetDirectoryContents(string.Empty)
.Where(f => f.IsDirectory == false)
.OrderBy(f => {
string[] fileNames = f.Name.Split('_');
int.TryParse(fileNames[0], out int index);
return index;
}))
{
await using Stream reader = fileInfo.CreateReadStream();
int fileLength = (int)fileInfo.Length;
byte[] newReadBytes = new byte[fileLength];
reader.Read(newReadBytes, 0, fileLength);
readBytes.AddRange(newReadBytes);
}
}
// Output to one file
await using FileStream stream = new FileStream($"{ActualFileDirectory}{fileName}", FileMode.Create);
await stream.WriteAsync(readBytes.ToArray(), 0, readBytes.Count);
// Delete chunks and directory
await DeleteTmpFilesAsync(tmpDirectoryPath);
return true;
}
private static void CreateRootDirectories()
{
if (Directory.Exists(ActualFileDirectory) == false)
{
DirectoryInfo info = Directory.CreateDirectory(ActualFileDirectory);
}
if (Directory.Exists(TmpFileDirectory) == false)
{
DirectoryInfo info = Directory.CreateDirectory(TmpFileDirectory);
}
}
private static string GetUniqueDirectoryPath(string rootDirectory, string original)
{
string directoryName = original;
if (Directory.Exists($"{rootDirectory}{directoryName}"))
{
int count = 0;
while (true)
{
if (Directory.Exists($"{rootDirectory}{directoryName}_{count.ToString()}"))
{
count += 1;
continue;
}
directoryName += count.ToString();
break;
}
}
return directoryName;
}
private static async Task DeleteTmpFilesAsync(string tmpDirectoryPath)
{
await Task.Run(() =>
{
using PhysicalFileProvider provider = new PhysicalFileProvider(tmpDirectoryPath);
foreach (IFileInfo fileInfo in provider.GetDirectoryContents(string.Empty))
{
if (fileInfo.IsDirectory)
{
Directory.Delete(fileInfo.PhysicalPath);
}
else
{
File.Delete(fileInfo.PhysicalPath);
}
}
Directory.Delete(tmpDirectoryPath);
});
}
}
}
コメント
コメントを投稿