FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

feat(storage): add support for 'skipIfExists' option for downloadMany by thiyaguk09 · Pull Request #2526 · googleapis/nodejs-storage · GitHub

This repository was archived by the owner on Mar 3, 2026. It is now read-only.
/ nodejs-storage Public archive
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension .ts  (2) All 1 file type selected
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
11 changes: 10 additions & 1 deletion src/transfer-manager.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import {
} from './file.js';
import pLimit from 'p-limit';
import * as path from 'path';
import {createReadStream, promises as fsp} from 'fs';
import {createReadStream, existsSync, promises as fsp} from 'fs';
import {CRC32C} from './crc32c.js';
import {GoogleAuth} from 'google-auth-library';
import {XMLParser, XMLBuilder} from 'fast-xml-parser';
Expand Down Expand Up @@ -108,6 +108,7 @@ export interface DownloadManyFilesOptions {
prefix?: string;
stripPrefix?: string;
passthroughOptions?: DownloadOptions;
skipIfExists?: boolean;
}

export interface DownloadFileInChunksOptions {
Expand Down Expand Up @@ -524,6 +525,8 @@ export class TransferManager {
* @property {string} [stripPrefix] A prefix to remove from all of the downloaded files.
* @property {object} [passthroughOptions] {@link DownloadOptions} Options to be passed through
* to each individual download operation.
* @property {boolean} [skipIfExists] Do not download the file if it already exists in
* the destination.
*
*/
/**
Expand Down Expand Up @@ -605,6 +608,12 @@ export class TransferManager {
if (options.stripPrefix) {
passThroughOptionsCopy.destination = file.name.replace(regex, '');
}
if (
options.skipIfExists &&
existsSync(passThroughOptionsCopy.destination || '')
) {
continue;
}

promises.push(
limit(async () => {
Expand Down
26 changes: 26 additions & 0 deletions test/transfer-manager.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import {
TransferManager,
Storage,
DownloadResponse,
DownloadManyFilesOptions,
} from '../src/index.js';
import assert from 'assert';
import * as path from 'path';
Expand Down Expand Up @@ -195,6 +196,10 @@ describe('Transfer Manager', () => {
});

describe('downloadManyFiles', () => {
beforeEach(() => {
sandbox.stub(fs, 'existsSync').returns(true);
});

it('calls download for each provided file', async () => {
let count = 0;
const firstFile = new File(bucket, 'first.txt');
Expand Down Expand Up @@ -276,6 +281,27 @@ describe('Transfer Manager', () => {
await transferManager.downloadManyFiles([file], {passthroughOptions});
});

it('does not download files that already exist locally when skipIfExists is true', async () => {
const firstFile = new File(bucket, 'first.txt');
sandbox.stub(firstFile, 'download').callsFake(options => {
assert.strictEqual(
(options as DownloadManyFilesOptions).skipIfExists,
0
);
});
const secondFile = new File(bucket, 'second.txt');
sandbox.stub(secondFile, 'download').callsFake(options => {
assert.strictEqual(
(options as DownloadManyFilesOptions).skipIfExists,
0
);
});

const files = [firstFile, secondFile];
const options = {skipIfExists: true};
await transferManager.downloadManyFiles(files, options);
});

it('does not set the destination when prefix, strip prefix and passthroughOptions.destination are not provided', async () => {
const options = {};
const filename = 'first.txt';
Expand Down

Back | FazBrowse Home | New Git URL