How To Download A File From Amazon S3 Bucket In Node.js Synchronously

I have to download multiple files from S3 bucket using node.js. For that I have to write a for loop & call the s3.getObject(param) method to download. After the files are downl

Solution 1:

Promises is better way,

var getObject = function(keyFile) {
    returnnewPromise(function(success, reject) {
        s3.getObject(
            { Bucket: "my-bucket", Key: keyFile },
            function (error, data) {
                if(error) {
                    reject(error);
                } else {
                    success(data);
                }
            }
        );
    });
}

var promises = [];
var fileContentList = newArrayList();

for(i=0; i<fileNameList.length i++){
    promises.push(getObject(fileNameList.get(i)));
}

Promise.all(promises)
.then(function(results) {
    for(var index in results) {
        var data = results[index];
        fileContentList.add(data.Body.toString());
    }
    // continue your process here
})
.catch(function(err) {
    alert(err);
});

Solution 2:

I have solved using this. Though I have not tried the answers of Alexander,Lena & Sébastian I believe each of the answers mentioned by them would also work in this case. Many many thanks to them for their quick reply:

Async.eachSeries(casCustomersList, function(customerName, next){
        if(casCustomersList.length>0 && customerName != customerId) {

            var paramToAws = {
                Bucket: bucketName,
                Key: folderPath +'applicationContext-security-' + customerName + '.xml'      //file name
            };
            AWSFileAccessManager.downloadFile(paramToAws, function(error, result) {
                if (error) {
                    next(error);
                } else {
                    customerApplicationContext.add(result.Body.toString());
                    next();
                }

            });
        } else{
            next();
        }

    }, function(err) {
       //Write the rest of your logic here to process synchronously as it is the callback function

    }

Solution 3:

Keep track of downloads you initiated in a separate list and as each download completes check to see if they are all done.

var fileContentList = new ArrayList();
var completedList = new ArrayList();
// function to setDone and initiate merge if all download attempts completed.function setDone(i) {
    completedList[i]=true;
    var allDone= true;
    for(i=0; i<completedList.length && allDone=completedList[i] && allDone; i++);
    if(allDone) {
       mergeFiles();
    }
}

// fill completedList with a false value for each fill to be downloadedfor(i=0; i<fileNameList.length;i++) completedList.add(false);

// initiate the downloadsfor(i=0; i<fileNameList.length; i++){
    s3.getObject({ Bucket: "my-bucket", Key: fileNameList.get(i) }, function (error, data) {
    if (error != null) {
      alert("Failed to retrieve an object: " + error);
    } else {
      alert("Loaded " + data.ContentLength + " bytes");
      fileContentList.add(data.Body.toString());
    }
    setDone(i);
  }
);
}

A more elegant solution, if you only want to merge the files if all downloads complete successfully:

var fileContentList = new ArrayList();

for(i=0; i<fileNameList.length i++){
    s3.getObject({ Bucket: "my-bucket", Key: fileNameList.get(i) }, function (error, data) {
    if (error != null) {
      alert("Failed to retrieve an object: " + error);
    } else {
      alert("Loaded " + data.ContentLength + " bytes");
      fileContentList.add(data.Body.toString());
    }
    if(fileContentList.length==fileNameList.length) combineFiles();
  }
);
}

Solution 4:

You can use here async each, it will download all the files in parallel. In this example the download will continue if some of the files failed, if you want to stop downloading the files once an error occurred, call the callback with the error, this will call to the final callback immediately.

async documentation

varasync = require('async');
var fileContentList = newArrayList();
functiondownloadS3Multiple(done){
    async.each([
            function (callback) {
                s3.getObject({Bucket: "my-bucket", Key: fileNameList.get(i)}, function (err, res) {
                    if (err) {
                        alert("Failed to retrieve an object: " + error);
                        callback();
                    }
                    else {
                        alert("Loaded " + data.ContentLength + " bytes");
                        fileContentList.add(data.Body.toString());
                        callback();

                    }
                })
            }
        ], function (err, results) {
            done(err, fileContentList)
        });
}