diff --git a/README.md b/README.md index 8605077fb..73aa2513e 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,7 @@ Samples are in the [`samples/`](/~https://github.com/googleapis/nodejs-storage/tre | Download Byte Range | [source code](/~https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadByteRange.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=/~https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadByteRange.js,samples/README.md) | | Download Encrypted File | [source code](/~https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadEncryptedFile.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=/~https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadEncryptedFile.js,samples/README.md) | | Download File | [source code](/~https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadFile.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=/~https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadFile.js,samples/README.md) | -| Download a File in Chunks Utilzing Transfer Manager | [source code](/~https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadFileInChunksWithTransferManager.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=/~https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadFileInChunksWithTransferManager.js,samples/README.md) | +| Download a File in Chunks With Transfer Manager | [source code](/~https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadFileInChunksWithTransferManager.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=/~https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadFileInChunksWithTransferManager.js,samples/README.md) | | Download File Using Requester Pays | [source code](/~https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadFileUsingRequesterPays.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=/~https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadFileUsingRequesterPays.js,samples/README.md) | | Download Folder With Transfer Manager | [source code](/~https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadFolderWithTransferManager.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=/~https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadFolderWithTransferManager.js,samples/README.md) | | Download Into Memory | [source code](/~https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadIntoMemory.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=/~https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadIntoMemory.js,samples/README.md) | @@ -209,6 +209,7 @@ Samples are in the [`samples/`](/~https://github.com/googleapis/nodejs-storage/tre | Upload Directory With Transfer Manager | [source code](/~https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadDirectoryWithTransferManager.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=/~https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadDirectoryWithTransferManager.js,samples/README.md) | | Upload Encrypted File | [source code](/~https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadEncryptedFile.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=/~https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadEncryptedFile.js,samples/README.md) | | Upload File | [source code](/~https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadFile.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=/~https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadFile.js,samples/README.md) | +| Upload a File in Chunks With Transfer Manager | [source code](/~https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadFileInChunksWithTransferManager.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=/~https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadFileInChunksWithTransferManager.js,samples/README.md) | | Upload File With Kms Key | [source code](/~https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadFileWithKmsKey.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=/~https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadFileWithKmsKey.js,samples/README.md) | | Upload From Memory | [source code](/~https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadFromMemory.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=/~https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadFromMemory.js,samples/README.md) | | Upload Many Files With Transfer Manager | [source code](/~https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadManyFilesWithTransferManager.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=/~https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadManyFilesWithTransferManager.js,samples/README.md) | diff --git a/samples/README.md b/samples/README.md index 7c775b9cb..10d611bc0 100644 --- a/samples/README.md +++ b/samples/README.md @@ -50,7 +50,7 @@ objects to users via direct download. * [Download Byte Range](#download-byte-range) * [Download Encrypted File](#download-encrypted-file) * [Download File](#download-file) - * [Download a File in Chunks Utilzing Transfer Manager](#download-a-file-in-chunks-utilzing-transfer-manager) + * [Download a File in Chunks With Transfer Manager](#download-a-file-in-chunks-with-transfer-manager) * [Download File Using Requester Pays](#download-file-using-requester-pays) * [Download Folder With Transfer Manager](#download-folder-with-transfer-manager) * [Download Into Memory](#download-into-memory) @@ -128,6 +128,7 @@ objects to users via direct download. * [Upload Directory With Transfer Manager](#upload-directory-with-transfer-manager) * [Upload Encrypted File](#upload-encrypted-file) * [Upload File](#upload-file) + * [Upload a File in Chunks With Transfer Manager](#upload-a-file-in-chunks-with-transfer-manager) * [Upload File With Kms Key](#upload-file-with-kms-key) * [Upload From Memory](#upload-from-memory) * [Upload Many Files With Transfer Manager](#upload-many-files-with-transfer-manager) @@ -718,7 +719,7 @@ __Usage:__ -### Download a File in Chunks Utilzing Transfer Manager +### Download a File in Chunks With Transfer Manager Downloads a single file in in chunks in parallel utilizing transfer manager. @@ -2086,6 +2087,25 @@ __Usage:__ +### Upload a File in Chunks With Transfer Manager + +Uploads a single file in in chunks in parallel utilizing transfer manager. + +View the [source code](/~https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadFileInChunksWithTransferManager.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=/~https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadFileInChunksWithTransferManager.js,samples/README.md) + +__Usage:__ + + +`node uploadFileInChunksWithTransferManager.js ` + + +----- + + + + ### Upload File With Kms Key View the [source code](/~https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadFileWithKmsKey.js). diff --git a/samples/downloadFileInChunksWithTransferManager.js b/samples/downloadFileInChunksWithTransferManager.js index f54be7a77..5fe1e64cd 100644 --- a/samples/downloadFileInChunksWithTransferManager.js +++ b/samples/downloadFileInChunksWithTransferManager.js @@ -12,14 +12,14 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * @experimental + * */ const path = require('path'); const cwd = path.join(__dirname, '..'); // sample-metadata: -// title: Download a File in Chunks Utilzing Transfer Manager +// title: Download a File in Chunks With Transfer Manager // description: Downloads a single file in in chunks in parallel utilizing transfer manager. // usage: node downloadFileInChunksWithTransferManager.js diff --git a/samples/downloadFolderWithTransferManager.js b/samples/downloadFolderWithTransferManager.js index 291a76003..9087f1f71 100644 --- a/samples/downloadFolderWithTransferManager.js +++ b/samples/downloadFolderWithTransferManager.js @@ -12,7 +12,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * @experimental + * */ // sample-metadata: diff --git a/samples/downloadManyFilesWithTransferManager.js b/samples/downloadManyFilesWithTransferManager.js index 15d8812b3..7a464ad4c 100644 --- a/samples/downloadManyFilesWithTransferManager.js +++ b/samples/downloadManyFilesWithTransferManager.js @@ -12,7 +12,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * @experimental + * */ // sample-metadata: diff --git a/samples/system-test/transfer-manager.test.js b/samples/system-test/transfer-manager.test.js index 919d4c77f..f6180bed3 100644 --- a/samples/system-test/transfer-manager.test.js +++ b/samples/system-test/transfer-manager.test.js @@ -80,6 +80,16 @@ describe('transfer manager', () => { ); }); + it('should upload a file utilizing chunked upload', async () => { + const output = execSync( + `node uploadFileInChunksWithTransferManager.js ${bucketName} ${firstFilePath} ${chunkSize}` + ); + assert.match( + output, + new RegExp(`${firstFilePath} uploaded to ${bucketName}.`) + ); + }); + it('should upload a directory', async () => { const output = execSync( `node uploadDirectoryWithTransferManager.js ${bucketName} ${resourcesPath}` diff --git a/samples/uploadDirectoryWithTransferManager.js b/samples/uploadDirectoryWithTransferManager.js index 37dd44214..28f29ec0e 100644 --- a/samples/uploadDirectoryWithTransferManager.js +++ b/samples/uploadDirectoryWithTransferManager.js @@ -12,7 +12,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * @experimental + * */ // sample-metadata: diff --git a/samples/uploadFileInChunksWithTransferManager.js b/samples/uploadFileInChunksWithTransferManager.js new file mode 100644 index 000000000..6bedd79ab --- /dev/null +++ b/samples/uploadFileInChunksWithTransferManager.js @@ -0,0 +1,67 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// sample-metadata: +// title: Upload a File in Chunks With Transfer Manager +// description: Uploads a single file in in chunks in parallel utilizing transfer manager. +// usage: node uploadFileInChunksWithTransferManager.js + +function main( + bucketName = 'my-bucket', + filePath = './local/path/to/file.txt', + chunkSize = 32 * 1024 * 1024 +) { + // [START storage_transfer_manager_upload_chunks_concurrently] + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // The ID of your GCS bucket + // const bucketName = 'your-unique-bucket-name'; + + // The path of file to upload + // const fileName = 'path/to/your/file'; + + // The size of each chunk to be uploaded + // const chunkSize = 32 * 1024 * 1024; + + // Imports the Google Cloud client library + const {Storage, TransferManager} = require('@google-cloud/storage'); + + // Creates a client + const storage = new Storage(); + + // Creates a transfer manager client + const transferManager = new TransferManager(storage.bucket(bucketName)); + + async function uploadFileInChunksWithTransferManager() { + // Uploads the files + await transferManager.uploadFileInChunks(filePath, { + chunkSizeBytes: chunkSize, + }); + + console.log(`${filePath} uploaded to ${bucketName}.`); + } + + uploadFileInChunksWithTransferManager().catch(console.error); + // [END storage_transfer_manager_upload_chunks_concurrently] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); +main(...process.argv.slice(2)); diff --git a/samples/uploadManyFilesWithTransferManager.js b/samples/uploadManyFilesWithTransferManager.js index 80054f4ff..a361abd05 100644 --- a/samples/uploadManyFilesWithTransferManager.js +++ b/samples/uploadManyFilesWithTransferManager.js @@ -12,7 +12,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * @experimental + * */ // sample-metadata: diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index b268b8339..a60fe2c39 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -33,39 +33,39 @@ const packageJson = require('../../package.json'); /** * Default number of concurrently executing promises to use when calling uploadManyFiles. - * @experimental + * */ -const DEFAULT_PARALLEL_UPLOAD_LIMIT = 2; +const DEFAULT_PARALLEL_UPLOAD_LIMIT = 5; /** * Default number of concurrently executing promises to use when calling downloadManyFiles. - * @experimental + * */ -const DEFAULT_PARALLEL_DOWNLOAD_LIMIT = 2; +const DEFAULT_PARALLEL_DOWNLOAD_LIMIT = 5; /** * Default number of concurrently executing promises to use when calling downloadFileInChunks. - * @experimental + * */ -const DEFAULT_PARALLEL_CHUNKED_DOWNLOAD_LIMIT = 2; +const DEFAULT_PARALLEL_CHUNKED_DOWNLOAD_LIMIT = 5; /** * The minimum size threshold in bytes at which to apply a chunked download strategy when calling downloadFileInChunks. - * @experimental + * */ const DOWNLOAD_IN_CHUNKS_FILE_SIZE_THRESHOLD = 32 * 1024 * 1024; /** * The chunk size in bytes to use when calling downloadFileInChunks. - * @experimental + * */ -const DOWNLOAD_IN_CHUNKS_DEFAULT_CHUNK_SIZE = 10 * 1024 * 1024; +const DOWNLOAD_IN_CHUNKS_DEFAULT_CHUNK_SIZE = 32 * 1024 * 1024; /** * The chunk size in bytes to use when calling uploadFileInChunks. - * @experimental + * */ const UPLOAD_IN_CHUNKS_DEFAULT_CHUNK_SIZE = 32 * 1024 * 1024; /** * Default number of concurrently executing promises to use when calling uploadFileInChunks. - * @experimental + * */ -const DEFAULT_PARALLEL_CHUNKED_UPLOAD_LIMIT = 2; +const DEFAULT_PARALLEL_CHUNKED_UPLOAD_LIMIT = 5; const EMPTY_REGEX = '(?:)'; @@ -165,7 +165,7 @@ export class MultiPartUploadError extends Error { * Class representing an implementation of MPU in the XML API. This class is not meant for public usage. * * @private - * @experimental + * */ class XMLMultiPartUploadHelper implements MultiPartUploadHelper { public partsMap; @@ -383,7 +383,7 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper { * @hideconstructor * * @param {Bucket} bucket A {@link Bucket} instance - * @experimental + * */ export class TransferManager { bucket: Bucket; @@ -400,7 +400,7 @@ export class TransferManager { * @property {string} [prefix] A prefix to append to all of the uploaded files. * @property {object} [passthroughOptions] {@link UploadOptions} Options to be passed through * to each individual upload operation. - * @experimental + * */ /** * Upload multiple files in parallel to the bucket. This is a convenience method @@ -429,7 +429,7 @@ export class TransferManager { * const response = await transferManager.uploadManyFiles('/local/directory'); * // Your bucket will now contain all files contained in '/local/directory' maintaining the subdirectory structure. * ``` - * @experimental + * */ async uploadManyFiles( filePathsOrDirectory: string[] | string, @@ -502,7 +502,7 @@ export class TransferManager { * @property {string} [stripPrefix] A prefix to remove from all of the downloaded files. * @property {object} [passthroughOptions] {@link DownloadOptions} Options to be passed through * to each individual download operation. - * @experimental + * */ /** * Download multiple files in parallel to the local filesystem. This is a convenience method @@ -534,7 +534,7 @@ export class TransferManager { * const response = await transferManager.downloadManyFiles('test-folder'); * // All files with GCS prefix of 'test-folder' have been downloaded. * ``` - * @experimental + * */ async downloadManyFiles( filesOrFolder: File[] | string[] | string, @@ -594,7 +594,7 @@ export class TransferManager { * to use when downloading the file. * @property {number} [chunkSizeBytes] The size in bytes of each chunk to be downloaded. * @property {string | boolean} [validation] Whether or not to perform a CRC32C validation check when download is complete. - * @experimental + * */ /** * Download a large file in chunks utilizing parallel download operations. This is a convenience method @@ -618,7 +618,7 @@ export class TransferManager { * // Your local directory now contains: * // - "large-file.txt" (with the contents from my-bucket.large-file.txt) * ``` - * @experimental + * */ async downloadFileInChunks( fileOrName: File | string, @@ -645,7 +645,7 @@ export class TransferManager { let start = 0; const filePath = options.destination || path.basename(file.name); - const fileToWrite = await fsp.open(filePath, 'w+'); + const fileToWrite = await fsp.open(filePath, 'w'); while (start < size) { const chunkStart = start; let chunkEnd = start + chunkSize - 1; @@ -704,7 +704,7 @@ export class TransferManager { * See {@link https://cloud.google.com/storage/docs/xml-api/post-object-multipart#request_headers| Request Headers: Initiate a Multipart Upload} * @property {boolean} [autoAbortFailure] boolean to indicate if an in progress upload session will be automatically aborted upon failure. If not set, * failures will be automatically aborted. - * @experimental + * */ /** * Upload a large file in chunks utilizing parallel upload opertions. If the upload fails, an uploadId and @@ -731,7 +731,7 @@ export class TransferManager { * // - "large-file.txt" * ``` * - * @experimental + * */ async uploadFileInChunks( filePath: string,