diff --git a/src/tasks/LargeFileUploadTask.ts b/src/tasks/LargeFileUploadTask.ts new file mode 100644 index 0000000..86819c1 --- /dev/null +++ b/src/tasks/LargeFileUploadTask.ts @@ -0,0 +1,331 @@ +/** + * ------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the MIT License. + * See License in the project root for license information. + * ------------------------------------------------------------------------------------------- + */ + +/** + * @module LargeFileUploadTask + **/ + +import { + Parsable, + RequestAdapter, + RequestInformation, + ParsableFactory, + ParseNode, + ErrorMappings, + HttpMethod, +} from "@microsoft/kiota-abstractions"; +import { UploadSlice } from "./UploadSlice"; +import { SeekableStreamReader } from "./SeekableStreamReader"; + +/** + * @interface + * Signature to represent progress receiver + * @property {number} progress - The progress value (This is the last uploaded byte) + */ +export interface IProgress { + report(progress: number): void; +} + +/** + * @interface + * Signature to represent an upload session, i.e the response returned by the server after for a pending upload + * + * @property {Date} expirationDateTime - The expiration time of the session + * @property {string[]} nextExpectedRanges - The next expected ranges + * @property {string} odataType - The type of the object + * @property {string} uploadUrl - The URL to which the file upload needs to be done + */ +export interface UploadSession { + expirationDateTime?: Date | null; + nextExpectedRanges?: string[] | null; + odataType?: string | null; + uploadUrl?: string | null; +} + +/** + * @interface + * Signature to represent the result of an upload + */ +export interface UploadResult { + itemResponse?: T | null; + location?: string; +} + +/** + * UploadSession ParsableFactory + * Creates a factory function to deserialize the upload session response. + * + * @param {ParseNode} _parseNode - The parse node to deserialize. + * @returns {Function} - A function that takes an instance of Parsable and returns a record of deserialization functions. + */ +export const createUploadSessionFromDiscriminatorValue = ( + _parseNode: ParseNode | undefined, +): ((instance?: Parsable) => Record void>) => { + return deserializeIntoUploadSession; +}; + +/** + * Deserializes the upload session response body. + * + * @param {Partial} [uploadSession] - The upload session object to deserialize into. + * @returns {Record void>} - A record of deserialization functions. + */ +export const deserializeIntoUploadSession = ( + uploadSession: Partial | undefined = {}, +): Record void> => { + return { + expirationDateTime: n => { + uploadSession.expirationDateTime = n.getDateValue(); + }, + nextExpectedRanges: n => { + uploadSession.nextExpectedRanges = n.getCollectionOfPrimitiveValues(); + }, + "@odata.type": n => { + uploadSession.odataType = n.getStringValue(); + }, + uploadUrl: n => { + uploadSession.uploadUrl = n.getStringValue(); + }, + }; +}; + +/** + * @constant + * A default slice size for a large file + */ +const DefaultSliceSize = 320 * 1024; + +/** + * A class representing LargeFileUploadTask + */ +export class LargeFileUploadTask { + /** + * @private + * The ranges to upload + */ + rangesRemaining: number[][] = []; + + /** + * @private + * The error mappings + */ + errorMappings: ErrorMappings; + + /** + * @private + * The seekable stream reader + */ + seekableStreamReader: SeekableStreamReader; + + /** + * @private + * The upload session + */ + Session: UploadSession; + + /** + * Constructs a new instance of the LargeFileUploadTask class. + * + * @param {RequestAdapter} requestAdapter - The request adapter to use for making HTTP requests. + * @param {Parsable} uploadSession - The upload session information. + * @param {ReadableStream} uploadStream - Returns an instance of an unconsumed new stream to be uploaded. + * @param {number} [maxSliceSize=-1] - The maximum size of each file slice to be uploaded. + * @param {ParsableFactory} parsableFactory - The factory to create parsable objects. + * @param {ErrorMappings} [errorMappings] - error mappings. + * + * @throws {Error} If any of the required parameters are undefined or invalid. + */ + constructor( + private readonly requestAdapter: RequestAdapter, + uploadSession: Parsable, + uploadStream: ReadableStream, + private readonly maxSliceSize = -1, + private readonly parsableFactory: ParsableFactory, + errorMappings: ErrorMappings, + ) { + if (!uploadSession) { + const error = new Error("Upload session is undefined, Please provide a valid upload session"); + error.name = "Invalid Upload Session Error"; + throw error; + } + if (!uploadStream) { + const error = new Error("Upload stream is undefined, Please provide a valid upload stream"); + error.name = "Invalid Upload Stream Error"; + throw error; + } + if (!requestAdapter) { + const error = new Error("Request adapter is undefined, Please provide a valid request adapter"); + error.name = "Invalid Request Adapter Error"; + throw error; + } + if (!parsableFactory) { + const error = new Error("Parsable factory is undefined, Please provide a valid parsable factory"); + error.name = "Invalid Parsable Factory Error"; + throw error; + } + if (uploadStream.locked) { + const error = new Error("Upload stream is locked, Please provide a valid upload stream"); + error.name = "Invalid Upload Stream Error"; + throw error; + } + if (maxSliceSize <= 0) { + this.maxSliceSize = DefaultSliceSize; + } + this.parsableFactory = parsableFactory; + + this.seekableStreamReader = new SeekableStreamReader(uploadStream); + + this.Session = this.extractSessionInfo(uploadSession); + this.rangesRemaining = this.getRangesRemaining(this.Session); + this.errorMappings = errorMappings; + } + + /** + * Uploads the file in a sequential order by slicing the file in terms of ranges. + * + * @param {IProgress} [progress] - Optional progress receiver to report upload progress. + * @returns {Promise>} - The result of the upload. + * @throws {Error} If the upload fails. + */ + public async upload(progress?: IProgress): Promise> { + const uploadUrl = this.Session.uploadUrl; + if (!uploadUrl) { + throw new Error("Upload URL is a required parameter."); + } + for (const range of this.rangesRemaining) { + let currentRangeBegin = range[0]; + while (currentRangeBegin <= range[1]) { + const nextSliceSize = this.nextSliceSize(currentRangeBegin, range[1]); + const uploadRequest = new UploadSlice( + this.requestAdapter, + uploadUrl, + currentRangeBegin, + currentRangeBegin + nextSliceSize - 1, + range[1] + 1, + this.parsableFactory, + this.errorMappings, + this.seekableStreamReader, + ); + const uploadResult = await uploadRequest.uploadSlice(); + progress?.report(uploadRequest.rangeEnd); + const { itemResponse, location } = uploadResult as Partial>; + if (itemResponse || location) { + return uploadResult as UploadResult; + } + currentRangeBegin += nextSliceSize; + } + } + throw new Error("Upload failed"); + } + + /** + * @public + * Resumes the current upload session + * @param progress + */ + public async resume(progress?: IProgress): Promise> { + await this.updateSession(); + return this.upload(progress); + } + + /** + * Refreshes the current upload session status by making a GET request to the upload URL. + * Updates the session expiration date, next expected ranges, and remaining ranges based on the response. + * + * @returns {Promise} - A promise that resolves to the updated upload session. + * @throws {Error} If the request fails. + */ + public async updateSession(): Promise { + const url = this.Session.uploadUrl; + if (!url) { + throw new Error("Upload url is invalid"); + } + const requestInformation = new RequestInformation(HttpMethod.GET, url); + const response = await this.requestAdapter.send( + requestInformation, + createUploadSessionFromDiscriminatorValue, + this.errorMappings, + ); + + if (response) { + this.Session.expirationDateTime = response.expirationDateTime; + this.Session.nextExpectedRanges = response.nextExpectedRanges; + if (response.uploadUrl) { + this.Session.uploadUrl = response.uploadUrl; + } + this.rangesRemaining = this.getRangesRemaining(this.Session); + } + return response; + } + + /** + * Deletes the current upload session. + * Sends a PUT request to the upload URL to cancel the session. + * + * @returns {Promise} A promise that resolves when the session is canceled. + */ + public async deleteSession(): Promise { + const url = this.Session.uploadUrl; + if (!url) { + throw new Error("Upload url is invalid"); + } + const requestInformation = new RequestInformation(HttpMethod.DELETE, url); + await this.requestAdapter.sendNoResponseContent(requestInformation, this.errorMappings); + } + + /** + * Extracts the upload session information from a parsable object. + * + * @param {Parsable} parsable - The parsable object containing the upload session information. + * @returns {UploadSession} - The extracted upload session information. + */ + private extractSessionInfo(parsable: Parsable): UploadSession { + const { expirationDateTime, nextExpectedRanges, odataType, uploadUrl } = parsable as Partial; + if (!nextExpectedRanges || !uploadUrl || nextExpectedRanges.length === 0) { + throw new Error("Upload session is invalid"); + } + return { + expirationDateTime: expirationDateTime ?? null, + nextExpectedRanges: nextExpectedRanges ?? null, + odataType: odataType ?? null, + uploadUrl: uploadUrl ?? null, + }; + } + + /** + * Calculates the size of the next slice to be uploaded. + * + * @param {number} currentRangeBegin - The beginning of the current range. + * @param {number} currentRangeEnd - The end of the current range. + * @returns {number} - The size of the next slice. + */ + private nextSliceSize(currentRangeBegin: number, currentRangeEnd: number): number { + const sizeBasedOnRange = currentRangeEnd - currentRangeBegin + 1; + return sizeBasedOnRange > this.maxSliceSize ? this.maxSliceSize : sizeBasedOnRange; + } + + /** + * @private + * Parses the upload session response and returns a nested number array of ranges pending upload + * @param uploadSession + */ + private getRangesRemaining(uploadSession: UploadSession): number[][] { + // nextExpectedRanges: https://dev.onedrive.com/items/upload_large_files.htm + // Sample: ["12345-55232","77829-99375"] + // Also, second number in range can be blank, which means 'until the end' + const ranges: number[][] = []; + if (uploadSession.nextExpectedRanges) { + ranges.push( + ...uploadSession.nextExpectedRanges.map(rangeString => { + const rangeArray = rangeString.split("-"); + return [parseInt(rangeArray[0], 10), parseInt(rangeArray[1], 10)]; + }), + ); + } + return ranges; + } +} diff --git a/src/tasks/SeekableStreamReader.ts b/src/tasks/SeekableStreamReader.ts new file mode 100644 index 0000000..dc0c82f --- /dev/null +++ b/src/tasks/SeekableStreamReader.ts @@ -0,0 +1,104 @@ +/** + * ------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the MIT License. + * See License in the project root for license information. + * ------------------------------------------------------------------------------------------- + */ + +/** + * @class + * + * A class that provides seekable read access to a `ReadableStream` of `Uint8Array`. + * This class allows reading specific sections of a stream without seeking backwards. + */ +export class SeekableStreamReader { + private readonly reader: ReadableStreamDefaultReader; + private cachedChunk?: Uint8Array | null; + private cachedPosition = 0; + private cachedOffset = 0; // Track where we are within the cached chunk + + /** + * Creates an instance of SeekableStreamReader. + * @param {ReadableStream} stream - The readable stream to read from. + */ + constructor(private readonly stream: ReadableStream) { + this.reader = stream.getReader(); + } + + /** + * Reads a section of the stream from the specified start position to the end position. + * + * This method reads data from the stream starting at the `start` position and ending at the `end` position. + * It ensures that the read operation does not seek backwards and handles chunked reading from the stream. + * The read data is collected into a single `ArrayBuffer` and returned. + * + * @param {number} start - The starting position of the section to read. Must be non-negative. + * @param {number} end - The ending position of the section to read. Must be greater than the start position. + * @returns {Promise} A promise that resolves to an `ArrayBuffer` containing the read section. + * @throws {Error} If the start position is negative, the end position is not greater than the start position, or if seeking backwards. + */ + public async readSection(start: number, end: number): Promise { + if (start < 0 || end <= start) { + throw new Error("Invalid range: start must be non-negative and end must be greater than start."); + } + if (start < this.cachedPosition) { + throw new Error("Cannot seek backwards."); + } + + const chunks: Uint8Array[] = []; + let totalLength = 0; + let position = this.cachedPosition; + + while (position < end) { + if (!this.cachedChunk) { + const { done, value } = await this.reader.read(); + if (done) break; + if (!value) continue; + this.cachedChunk = value; + this.cachedOffset = 0; // Reset the offset since it's a new chunk + } + + const chunk = this.cachedChunk; + const chunkLength = chunk.byteLength; + + // Skip chunks that are entirely before `start` + if (position + chunkLength - this.cachedOffset <= start) { + position += chunkLength - this.cachedOffset; + this.cachedChunk = null; + this.cachedOffset = 0; + continue; + } + + // Calculate the section of the chunk to return + const startIndex = Math.max(0, start - position - 1) + this.cachedOffset; + const endIndex = Math.min(chunkLength, end - position + this.cachedOffset + 1); + + const sliced = chunk.slice(startIndex, endIndex); + chunks.push(sliced); + totalLength += sliced.byteLength; + + // Update position correctly + position += sliced.byteLength; + + // Store remaining unread data for future reads + if (endIndex < chunkLength) { + this.cachedOffset = endIndex; + } else { + this.cachedChunk = null; + this.cachedOffset = 0; + } + } + + this.cachedPosition = position; // Ensure position is updated + + // Combine collected chunks into a single buffer + const result = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.byteLength; + } + + return result.buffer; + } +} diff --git a/src/tasks/UploadSlice.ts b/src/tasks/UploadSlice.ts new file mode 100644 index 0000000..9a8d4fd --- /dev/null +++ b/src/tasks/UploadSlice.ts @@ -0,0 +1,109 @@ +/** + * ------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the MIT License. + * See License in the project root for license information. + * ------------------------------------------------------------------------------------------- + */ + +import { + ErrorMappings, + Headers, + HttpMethod, + Parsable, + ParsableFactory, + RequestAdapter, + RequestInformation, + type AdditionalDataHolder, +} from "@microsoft/kiota-abstractions"; +import { HeadersInspectionOptions } from "@microsoft/kiota-http-fetchlibrary"; +import { UploadResult, UploadSession } from "./LargeFileUploadTask"; +import { SeekableStreamReader } from "./SeekableStreamReader"; + +const binaryContentType = "application/octet-stream"; + +/** + * @class + * Represents a slice of a file to be uploaded. + * + * @template T - The type of the parsable object. + */ +export class UploadSlice { + /** + * Constructs an instance of the UploadSlice class. + * + * @param {RequestAdapter} requestAdapter - The request adapter to use for making HTTP requests. + * @param {string} sessionUrl - The URL of the upload session. + * @param {number} rangeBegin - The beginning byte position of the slice. + * @param {number} rangeEnd - The ending byte position of the slice. + * @param {number} totalSessionLength - The total length of the upload session. + * @param {ParsableFactory} parsableFactory - The factory to create parsable objects. + * @param {ErrorMappings} errorMappings - The mappings for handling errors. + * @param {SeekableStreamReader} seekableStreamReader - The stream reader to read the file slice. + */ + constructor( + readonly requestAdapter: RequestAdapter, + readonly sessionUrl: string, + readonly rangeBegin: number, + readonly rangeEnd: number, + readonly totalSessionLength: number, + readonly parsableFactory: ParsableFactory, + readonly errorMappings: ErrorMappings, + readonly seekableStreamReader: SeekableStreamReader, + ) {} + + /** + * Uploads a slice of the file to the server. + * + * @returns {Promise | undefined>} - The result of the upload operation. + */ + public async uploadSlice(): Promise | UploadSession | undefined> { + const data = await this.seekableStreamReader.readSection(this.rangeBegin, this.rangeEnd); + const requestInformation = new RequestInformation(HttpMethod.PUT, this.sessionUrl); + requestInformation.headers = new Headers([ + ["Content-Range", new Set([`bytes ${this.rangeBegin}-${this.rangeEnd}/${this.totalSessionLength}`])], + ["Content-Length", new Set([`${this.rangeEnd - this.rangeBegin}`])], + ]); + requestInformation.setStreamContent(data, binaryContentType); + + const headerOptions = new HeadersInspectionOptions({ inspectResponseHeaders: true }); + requestInformation.addRequestOptions([headerOptions]); + + const itemResponse = await this.requestAdapter.send( + requestInformation, + this.parsableFactory, + this.errorMappings, + ); + + const locations = headerOptions.getResponseHeaders().get("location"); + + if (itemResponse) { + let sessionResponse = this.isUploadSessionResponse(itemResponse); + if (sessionResponse) { + return sessionResponse; + } + const { additionalData } = itemResponse as Partial; + if (additionalData) { + sessionResponse = this.isUploadSessionResponse(additionalData); + if (sessionResponse) { + return sessionResponse; + } + } + } + + return { + itemResponse, + location: locations ? (locations as unknown as string[])[0] : undefined, + }; + } + + private isUploadSessionResponse(item: Parsable | AdditionalDataHolder): UploadSession | null { + const { expirationDateTime, nextExpectedRanges } = item as Partial; + if (nextExpectedRanges) { + return { + expirationDateTime, + nextExpectedRanges, + }; + } + return null; + } +} diff --git a/src/tasks/index.ts b/src/tasks/index.ts index 9b5b137..ea9c5d5 100644 --- a/src/tasks/index.ts +++ b/src/tasks/index.ts @@ -1 +1,4 @@ +export * from "./LargeFileUploadTask"; export * from "./PageIterator.js"; +export * from "./SeekableStreamReader.js"; +export * from "./UploadSlice.js"; diff --git a/test/task/LargeFileUploadTask.ts b/test/task/LargeFileUploadTask.ts new file mode 100644 index 0000000..29faabd --- /dev/null +++ b/test/task/LargeFileUploadTask.ts @@ -0,0 +1,309 @@ +import { assert, describe, it } from "vitest"; +import { IProgress, LargeFileUploadTask, SeekableStreamReader } from "../../src"; +// @ts-ignore +import { DummyRequestAdapter } from "../utils/DummyRequestAdapter"; +import { ErrorMappings, Parsable, ParseNode } from "@microsoft/kiota-abstractions"; +import { UploadSlice } from "../../src/tasks/UploadSlice"; + +const adapter = new DummyRequestAdapter(); + +interface SampleResponse extends Parsable { + nextExpectedRanges?: string[] | undefined; + expirationDateTime?: Date | undefined; + uploadUrl?: string | undefined; +} + +function createPageCollectionFromDiscriminatorValue( + parseNode: ParseNode | undefined, +): (instance?: Parsable) => Record void> { + return deserializeIntoPageCollection; +} + +function deserializeIntoPageCollection( + baseDeltaFunctionResponse: Partial | undefined = {}, +): Record void> { + return {}; +} + +function createSampleReadableStream(): ReadableStream { + return new ReadableStream({ + start: controller => { + const encoder = new TextEncoder(); + const chunk = encoder.encode("This is a 24-byte string"); + controller.enqueue(chunk); + controller.close(); + }, + }); +} + +const errorMappings: ErrorMappings = { + XXX: parseNode => createGraphErrorFromDiscriminatorValue(parseNode), +}; + +export const createGraphErrorFromDiscriminatorValue = ( + _parseNode: ParseNode | undefined, +): ((instance?: Parsable) => Record void>) => { + return deserializeIntoGraphError; +}; + +/** + * Deserializes the batch item + * @param graphError + */ +export const deserializeIntoGraphError = ( + graphError: Partial | undefined = {}, +): Record void> => { + return {}; +}; + +describe("LargeFileUploadTask tests", () => { + describe("initialization", () => { + it("should initialize", () => { + const session: SampleResponse = { + nextExpectedRanges: ["0-23"], + expirationDateTime: new Date(Date.now() + 24 * 60 * 60 * 1000), // + uploadUrl: "https://example.com/upload", + }; + const largeFileUploadTask = new LargeFileUploadTask( + adapter, + session, + createSampleReadableStream(), + 10, + createPageCollectionFromDiscriminatorValue, + errorMappings, + ); + assert.isNotNull(largeFileUploadTask, "LargeFileUploadTask failed to initialize"); + }); + it("should throw an error if invalid upload session is given", () => { + assert.throws(() => { + new LargeFileUploadTask( + adapter, + {} as SampleResponse, + createSampleReadableStream(), + 10, + createPageCollectionFromDiscriminatorValue, + errorMappings, + ); + }, "Upload session is invalid"); + }); + }); + describe("file upload", () => { + it("should split the file into expected ranges observing max size", async () => { + adapter.resetAdapter(); + const session: SampleResponse = { + nextExpectedRanges: ["0-23"], + expirationDateTime: new Date(Date.now() + 24 * 60 * 60 * 1000), // + uploadUrl: "https://example.com/upload", + }; + const largeFileUploadTask = new LargeFileUploadTask( + adapter, + session, + createSampleReadableStream(), + 5, + createPageCollectionFromDiscriminatorValue, + errorMappings, + ); + + adapter.setResponse( + { + nextExpectedRanges: ["10-19"], + expirationDateTime: new Date(Date.now() + 24 * 60 * 60 * 1000), + uploadUrl: "https://example.com/upload", + }, + { + nextExpectedRanges: ["10-19"], + expirationDateTime: new Date(Date.now() + 24 * 60 * 60 * 1000), + uploadUrl: "https://example.com/upload", + }, + { + nextExpectedRanges: ["10-19"], + expirationDateTime: new Date(Date.now() + 24 * 60 * 60 * 1000), + uploadUrl: "https://example.com/upload", + }, + { + nextExpectedRanges: ["22-23"], + expirationDateTime: new Date(Date.now() + 24 * 60 * 60 * 1000), + uploadUrl: "https://example.com/upload", + }, + { + name: "Valid", + }, + ); + + await largeFileUploadTask.upload(); + + // get all the request information from the adapter and rebuild the slice start and stop from the header + const requests = adapter.getRequests(); + + // get Content-Range and build start and end byte + const pairs: [number, number][] = requests.map(request => { + const ranges = request.headers.get("Content-Range"); + if (!ranges) { + throw new Error("Content-Range header is missing"); + } + const rangeValue = ranges.values().next().value; + if (!rangeValue) { + throw new Error("Content-Range value is missing"); + } + const range = rangeValue.split(" ")[1].split("/")[0]; + const [start, end] = range.split("-").map(Number); + return [start, end]; + }); + + // cast the arrays to UploadSlice + assert.equal(pairs.length, 5); + + pairs.forEach(slice => { + const [rangeBegin, rangeEnd] = slice; + assert.isAtMost(rangeEnd - rangeBegin + 1, 5, "Slice size should not be larger than 5"); + }); + + for (let i = 0; i < pairs.length - 1; i++) { + assert.isTrue(pairs[i][1] < pairs[i + 1][0], "Slices should be in order from largest to smallest"); + } + + const decoder = new TextDecoder(); + let reconstructedString = ""; + const seekableStream = new SeekableStreamReader(createSampleReadableStream()); + for (let i = 0; i < pairs.length; i++) { + const [rangeBegin, rangeEnd] = pairs[i]; + const chunk = await seekableStream.readSection(rangeBegin, rangeEnd); + reconstructedString += decoder.decode(chunk); + } + assert.equal(reconstructedString, "This is a 24-byte string", "Reconstructed string should match the original"); + }); + it("should execute multiple file upload requests", async () => { + adapter.resetAdapter(); + const session: SampleResponse = { + nextExpectedRanges: ["0-23"], + expirationDateTime: new Date(Date.now() + 24 * 60 * 60 * 1000), // + uploadUrl: "https://example.com/upload", + }; + const largeFileUploadTask = new LargeFileUploadTask( + adapter, + session, + createSampleReadableStream(), + 10, + createPageCollectionFromDiscriminatorValue, + errorMappings, + ); + + let progressCounter = 0; + let lastCall = -1; + const progressCallback: IProgress = { + report: (progress: number) => { + lastCall = progress; + progressCounter++; + }, + }; + + adapter.setResponse({ + nextExpectedRanges: ["10-19"], + expirationDateTime: new Date(Date.now() + 24 * 60 * 60 * 1000), // + uploadUrl: "https://example.com/upload", + }); + + adapter.setResponse({ + nextExpectedRanges: ["20-23"], + expirationDateTime: new Date(Date.now() + 24 * 60 * 60 * 1000), // + uploadUrl: "https://example.com/upload", + }); + adapter.setResponse({ + name: "Valid", + }); + + await largeFileUploadTask.upload(progressCallback); + assert.equal(progressCounter, 3); + assert.equal(lastCall, 23); + + const requests = adapter.getRequests(); + assert.equal(requests.length, 3); + }); + it("should delete an upload session", () => { + adapter.resetAdapter(); + const session: SampleResponse = { + nextExpectedRanges: ["0-23"], + expirationDateTime: new Date(Date.now() + 24 * 60 * 60 * 1000), // + uploadUrl: "https://example.com/upload", + }; + const largeFileUploadTask = new LargeFileUploadTask( + adapter, + session, + createSampleReadableStream(), + 10, + createPageCollectionFromDiscriminatorValue, + errorMappings, + ); + largeFileUploadTask.deleteSession(); + + // check if it makes a delete call to the url + const requests = adapter.getRequests(); + assert.equal(requests.length, 1); + assert.equal(requests[0].httpMethod, "DELETE"); + assert.equal(requests[0].URL, session.uploadUrl); + }); + it("should update an upload session", () => { + adapter.resetAdapter(); + const session: SampleResponse = { + nextExpectedRanges: ["0-23"], + expirationDateTime: new Date(Date.now() + 24 * 60 * 60 * 1000), // + uploadUrl: "https://example.com/upload", + }; + const largeFileUploadTask = new LargeFileUploadTask( + adapter, + session, + createSampleReadableStream(), + 10, + createPageCollectionFromDiscriminatorValue, + errorMappings, + ); + + largeFileUploadTask.updateSession(); + + // check if it makes a get call to the url + const requests = adapter.getRequests(); + assert.equal(requests.length, 1); + assert.equal(requests[0].httpMethod, "GET"); + assert.equal(requests[0].URL, session.uploadUrl); + }); + it("should resume an upload session", async () => { + adapter.resetAdapter(); + const session: SampleResponse = { + nextExpectedRanges: ["0-23"], + expirationDateTime: new Date(Date.now() + 24 * 60 * 60 * 1000), // + uploadUrl: "https://example.com/upload", + }; + const largeFileUploadTask = new LargeFileUploadTask( + adapter, + session, + createSampleReadableStream(), + 10, + createPageCollectionFromDiscriminatorValue, + errorMappings, + ); + + adapter.setResponse({ + nextExpectedRanges: ["20-39"], + expirationDateTime: new Date(Date.now() + 24 * 60 * 60 * 1000), // + uploadUrl: "https://example.com/upload", + }); + adapter.setResponse({ + name: "Valid", + }); + + // check if it first call to the url is a get for refreshing the session and then multiple uploads + await largeFileUploadTask.resume(); + const requests = adapter.getRequests(); + assert.equal(requests.length, 2); + assert.equal(requests[0].httpMethod, "GET"); + assert.equal(requests[0].URL, session.uploadUrl); + + let pos = 1; + while (requests.length > pos) { + assert.equal(requests[pos].httpMethod, "PUT"); + assert.equal(requests[pos].URL, session.uploadUrl); + pos++; + } + }); + }); +}); diff --git a/test/task/SeekableStreamReader.ts b/test/task/SeekableStreamReader.ts new file mode 100644 index 0000000..61d8e0c --- /dev/null +++ b/test/task/SeekableStreamReader.ts @@ -0,0 +1,45 @@ +import { assert, describe, it } from "vitest"; +import { SeekableStreamReader } from "../../src"; + +function createRandoReadableStream(value: string): ReadableStream { + return new ReadableStream({ + start: controller => { + const encoder = new TextEncoder(); + const chunk = encoder.encode(value); + controller.enqueue(chunk); + controller.close(); + }, + }); +} + +describe("SeekableStreamReader tests", () => { + it("should read the sections of the stream", async () => { + const value = veryLongRandomText; + const stream = createRandoReadableStream(value); + + const reader = new SeekableStreamReader(stream); + + // split the stream into pairs 20 pairs of start and end + const pairs = []; + const size = new TextEncoder().encode(veryLongRandomText).length; + const units = 4; + + const batchSize = Math.round(size / units); + for (let i = 0; i < value.length; i += batchSize) { + const start: number = pairs.length === 0 ? i : pairs[pairs.length - 1][1] + 1; + pairs.push([start, Math.min(start + batchSize, value.length)]); + } + + const decoder = new TextDecoder(); + let reconstructedString = ""; + for (const [start, end] of pairs) { + const section = await reader.readSection(start, end); + reconstructedString += decoder.decode(section); + } + assert.equal(value, reconstructedString, "Reconstructed string should match the original"); + }); +}); + +const veryLongRandomText = "This is a very long text that will be used to test the SeekableStreamReader class.".repeat( + 20, +); //.split('').sort(() => 0.5 - Math.random()).join(''); diff --git a/test/utils/DummyRequestAdapter.ts b/test/utils/DummyRequestAdapter.ts index 9416357..f1e1e25 100644 --- a/test/utils/DummyRequestAdapter.ts +++ b/test/utils/DummyRequestAdapter.ts @@ -46,8 +46,8 @@ export class DummyRequestAdapter implements RequestAdapter { } // set a fake response - setResponse(response: any): void { - this.response.push(response); + setResponse(...responses: any): void { + this.response.push(...responses); } // get requests @@ -55,6 +55,11 @@ export class DummyRequestAdapter implements RequestAdapter { return this.requests; } + resetAdapter(): void { + this.requests = []; + this.response = []; + } + convertToNativeRequest(requestInfo: RequestInformation): Promise { return Promise.resolve(undefined as T); }