| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
1 parent f2e1e0c commit 9f62429
4 files changed
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -2288,12 +2288,13 @@ class File extends ServiceObject<File, FileMetadata> { | |||
| 2288 | 2288 | ||
| 2289 | 2289 | const fileStream = this.createReadStream(options); | |
| 2290 | 2290 | let receivedData = false; | |
| 2291 | + | ||
| 2291 | 2292 | if (destination) { | |
| 2292 | 2293 | fileStream | |
| 2293 | 2294 | .on('error', callback) | |
| 2294 | 2295 | .once('data', data => { | |
| 2295 | - // We know that the file exists the server - now we can truncate/write to a file | ||
| 2296 | 2296 | receivedData = true; | |
| 2297 | + // We know that the file exists the server - now we can truncate/write to a file | ||
| 2297 | 2298 | const writable = fs.createWriteStream(destination); | |
| 2298 | 2299 | writable.write(data); | |
| 2299 | 2300 | fileStream | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -526,7 +526,9 @@ export class TransferManager { | |||
| 526 | 526 | * | |
| 527 | 527 | * @param {array | string} [filesOrFolder] An array of file name strings or file objects to be downloaded. If | |
| 528 | 528 | * a string is provided this will be treated as a GCS prefix and all files with that prefix will be downloaded. | |
| 529 | - * @param {DownloadManyFilesOptions} [options] Configuration options. | ||
| 529 | + * @param {DownloadManyFilesOptions} [options] Configuration options. Setting options.prefix or options.stripPrefix | ||
| 530 | + * or options.passthroughOptions.destination will cause the downloaded files to be written to the file system | ||
| 531 | + * instead of being returned as a buffer. | ||
| 530 | 532 | * @returns {Promise<DownloadResponse[]>} | |
| 531 | 533 | * | |
| 532 | 534 | * @example | |
@@ -587,7 +589,7 @@ export class TransferManager { | |||
| 587 | 589 | [GCCL_GCS_CMD_KEY]: GCCL_GCS_CMD_FEATURE.DOWNLOAD_MANY, | |
| 588 | 590 | }; | |
| 589 | 591 | ||
| 590 | - if (options.prefix) { | ||
| 592 | + if (options.prefix || passThroughOptionsCopy.destination) { | ||
| 591 | 593 | passThroughOptionsCopy.destination = path.join( | |
| 592 | 594 | options.prefix || '', | |
| 593 | 595 | passThroughOptionsCopy.destination || '', | |
@@ -598,7 +600,19 @@ export class TransferManager { | |||
| 598 | 600 | passThroughOptionsCopy.destination = file.name.replace(regex, ''); | |
| 599 | 601 | } | |
| 600 | 602 | ||
| 601 | - promises.push(limit(() => file.download(passThroughOptionsCopy))); | ||
| 603 | + promises.push( | ||
| 604 | + limit(async () => { | ||
| 605 | + const destination = passThroughOptionsCopy.destination; | ||
| 606 | + if (destination && destination.endsWith(path.sep)) { | ||
| 607 | + await fsp.mkdir(destination, {recursive: true}); | ||
| 608 | + return Promise.resolve([ | ||
| 609 | + Buffer.alloc(0), | ||
| 610 | + ]) as Promise<DownloadResponse>; | ||
| 611 | + } | ||
| 612 | + | ||
| 613 | + return file.download(passThroughOptionsCopy); | ||
| 614 | + }) | ||
| 615 | + ); | ||
| 602 | 616 | } | |
| 603 | 617 | ||
| 604 | 618 | return Promise.all(promises); | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -28,6 +28,7 @@ import assert from 'assert'; | |||
| 28 | 28 | import * as crypto from 'crypto'; | |
| 29 | 29 | import duplexify from 'duplexify'; | |
| 30 | 30 | import * as fs from 'fs'; | |
| 31 | + import * as path from 'path'; | ||
| 31 | 32 | import proxyquire from 'proxyquire'; | |
| 32 | 33 | import * as resumableUpload from '../src/resumable-upload.js'; | |
| 33 | 34 | import * as sinon from 'sinon'; | |
@@ -2571,6 +2572,12 @@ describe('File', () => { | |||
| 2571 | 2572 | }); | |
| 2572 | 2573 | ||
| 2573 | 2574 | describe('with destination', () => { | |
| 2575 | + const sandbox = sinon.createSandbox(); | ||
| 2576 | + | ||
| 2577 | + afterEach(() => { | ||
| 2578 | + sandbox.restore(); | ||
| 2579 | + }); | ||
| 2580 | + | ||
| 2574 | 2581 | it('should write the file to a destination if provided', done => { | |
| 2575 | 2582 | tmp.setGracefulCleanup(); | |
| 2576 | 2583 | tmp.file((err, tmpFilePath) => { | |
@@ -2694,6 +2701,29 @@ describe('File', () => { | |||
| 2694 | 2701 | }); | |
| 2695 | 2702 | }); | |
| 2696 | 2703 | }); | |
| 2704 | + | ||
| 2705 | + it('should fail if provided destination directory does not exist', done => { | ||
| 2706 | + tmp.setGracefulCleanup(); | ||
| 2707 | + tmp.dir(async (err, tmpDirPath) => { | ||
| 2708 | + assert.ifError(err); | ||
| 2709 | + | ||
| 2710 | + const fileContents = 'nested-abcdefghijklmnopqrstuvwxyz'; | ||
| 2711 | + | ||
| 2712 | + Object.assign(fileReadStream, { | ||
| 2713 | + _read(this: Readable) { | ||
| 2714 | + this.push(fileContents); | ||
| 2715 | + this.push(null); | ||
| 2716 | + }, | ||
| 2717 | + }); | ||
| 2718 | + | ||
| 2719 | + const nestedPath = path.join(tmpDirPath, 'a', 'b', 'c', 'file.txt'); | ||
| 2720 | + | ||
| 2721 | + file.download({destination: nestedPath}, (err: Error) => { | ||
| 2722 | + assert.ok(err); | ||
| 2723 | + done(); | ||
| 2724 | + }); | ||
| 2725 | + }); | ||
| 2726 | + }); | ||
| 2697 | 2727 | }); | |
| 2698 | 2728 | }); | |
| 2699 | 2729 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -19,6 +19,7 @@ import { | |||
| 19 | 19 | Bucket, | |
| 20 | 20 | File, | |
| 21 | 21 | CRC32C, | |
| 22 | + DownloadCallback, | ||
| 22 | 23 | DownloadOptions, | |
| 23 | 24 | IdempotencyStrategy, | |
| 24 | 25 | MultiPartHelperGenerator, | |
@@ -233,6 +234,78 @@ describe('Transfer Manager', () => { | |||
| 233 | 234 | ||
| 234 | 235 | await transferManager.downloadManyFiles([file]); | |
| 235 | 236 | }); | |
| 237 | + | ||
| 238 | + it('sets the destination correctly when provided a passthroughOptions.destination', async () => { | ||
| 239 | + const passthroughOptions = { | ||
| 240 | + destination: 'test-destination', | ||
| 241 | + }; | ||
| 242 | + const filename = 'first.txt'; | ||
| 243 | + const expectedDestination = path.normalize( | ||
| 244 | + `${passthroughOptions.destination}/${filename}` | ||
| 245 | + ); | ||
| 246 | + const download = (optionsOrCb?: DownloadOptions | DownloadCallback) => { | ||
| 247 | + if (typeof optionsOrCb === 'function') { | ||
| 248 | + optionsOrCb(null, Buffer.alloc(0)); | ||
| 249 | + } else if (optionsOrCb) { | ||
| 250 | + assert.strictEqual(optionsOrCb.destination, expectedDestination); | ||
| 251 | + } | ||
| 252 | + return Promise.resolve([Buffer.alloc(0)]) as Promise<DownloadResponse>; | ||
| 253 | + }; | ||
| 254 | + | ||
| 255 | + const file = new File(bucket, filename); | ||
| 256 | + file.download = download; | ||
| 257 | + await transferManager.downloadManyFiles([file], {passthroughOptions}); | ||
| 258 | + }); | ||
| 259 | + | ||
| 260 | + it('does not set the destination when prefix, strip prefix and passthroughOptions.destination are not provided', async () => { | ||
| 261 | + const options = {}; | ||
| 262 | + const filename = 'first.txt'; | ||
| 263 | + const download = (optionsOrCb?: DownloadOptions | DownloadCallback) => { | ||
| 264 | + if (typeof optionsOrCb === 'function') { | ||
| 265 | + optionsOrCb(null, Buffer.alloc(0)); | ||
| 266 | + } else if (optionsOrCb) { | ||
| 267 | + assert.strictEqual(optionsOrCb.destination, undefined); | ||
| 268 | + } | ||
| 269 | + return Promise.resolve([Buffer.alloc(0)]) as Promise<DownloadResponse>; | ||
| 270 | + }; | ||
| 271 | + | ||
| 272 | + const file = new File(bucket, filename); | ||
| 273 | + file.download = download; | ||
| 274 | + await transferManager.downloadManyFiles([file], options); | ||
| 275 | + }); | ||
| 276 | + | ||
| 277 | + it('should recursively create directory and write file contents if destination path is nested', async () => { | ||
| 278 | + const prefix = 'text-prefix'; | ||
| 279 | + const folder = 'nestedFolder/'; | ||
| 280 | + const file = 'first.txt'; | ||
| 281 | + const filesOrFolder = [folder, path.join(folder, file)]; | ||
| 282 | + const expectedFilePath = path.join(prefix, folder, file); | ||
| 283 | + const expectedDir = path.join(prefix, folder); | ||
| 284 | + const mkdirSpy = sandbox.spy(fsp, 'mkdir'); | ||
| 285 | + const download = (optionsOrCb?: DownloadOptions | DownloadCallback) => { | ||
| 286 | + if (typeof optionsOrCb === 'function') { | ||
| 287 | + optionsOrCb(null, Buffer.alloc(0)); | ||
| 288 | + } else if (optionsOrCb) { | ||
| 289 | + assert.strictEqual(optionsOrCb.destination, expectedFilePath); | ||
| 290 | + } | ||
| 291 | + return Promise.resolve([Buffer.alloc(0)]) as Promise<DownloadResponse>; | ||
| 292 | + }; | ||
| 293 | + | ||
| 294 | + sandbox.stub(bucket, 'file').callsFake(filename => { | ||
| 295 | + const file = new File(bucket, filename); | ||
| 296 | + file.download = download; | ||
| 297 | + return file; | ||
| 298 | + }); | ||
| 299 | + await transferManager.downloadManyFiles(filesOrFolder, { | ||
| 300 | + prefix: prefix, | ||
| 301 | + }); | ||
| 302 | + assert.strictEqual( | ||
| 303 | + mkdirSpy.calledOnceWith(expectedDir, { | ||
| 304 | + recursive: true, | ||
| 305 | + }), | ||
| 306 | + true | ||
| 307 | + ); | ||
| 308 | + }); | ||
| 236 | 309 | }); | |
| 237 | 310 | ||
| 238 | 311 | describe('downloadFileInChunks', () => { | |
| Back | FazBrowse Home | New Git URL |
0 commit comments