Node.js & Amazon S3: How to iterate through all files in a bucket?

Question

Is there any Amazon S3 client library for Node.js that allows listing of all files in S3 bucket?

The most known aws2js and knox don\'t seem to have this functionalit

Answer 1

Using the new API s3.listObjectsV2 the recursive solution will be:

S3Dataset.prototype.listFiles = function(params,callback) {
    var self=this;

    var options = {
    };
    for (var attrname in params) { options[attrname] = params[attrname]; }

    var results=[];
    var s3=self.s3Store.GetInstance();
    function listAllKeys(token, callback) {
        var opt={ Bucket: self._options.s3.Bucket, Prefix: self._options.s3.Key, MaxKeys: 1000 };
        if(token) opt.ContinuationToken = token;
        s3.listObjectsV2(opt, (error, data) => {
            if (error) {
                if(self.logger) this.logger.error("listFiles error:", error);
                return callback(error);
            } else {
                for (var index in data.Contents) {
                    var bucket = data.Contents[index];
                    if(self.logger) self.logger.debug("listFiles Key: %s LastModified: %s Size: %s", bucket.Key, bucket.LastModified, bucket.Size);
                    if(bucket.Size>0) {
                        var Bucket=self._options.s3.Bucket;
                        var Key=bucket.Key;
                        var components=bucket.Key.split('/');
                        var name=components[components.length-1];
                        results.push({
                            name: name,
                            path: bucket.Key,
                            mtime: bucket.LastModified,
                            size: bucket.Size,
                            sizehr: formatSizeUnits(bucket.Size)
                        });
                    }
                }
                if( data.IsTruncated ) { // truncated page
                    return listAllKeys(data.NextContinuationToken, callback);
                } else {
                    return callback(null,results);
                }
            }
        });
    }
    return listAllKeys.apply(this,['',callback]);
};

where

function formatSizeUnits(bytes){
    if      (bytes>=1099511627776) {bytes=(bytes/1099511627776).toFixed(4)+' PB';}
    else if (bytes>=1073741824)    {bytes=(bytes/1073741824).toFixed(4)+' GB';}
    else if (bytes>=1048576)       {bytes=(bytes/1048576).toFixed(4)+' MB';}
    else if (bytes>=1024)          {bytes=(bytes/1024).toFixed(4)+' KB';}
    else if (bytes>1)              {bytes=bytes+' bytes';}
    else if (bytes==1)             {bytes=bytes+' byte';}
    else                           {bytes='0 byte';}
    return bytes;
}//formatSizeUnits

Answer 2

I made it as simple as possible. You can iterate uploading objects using for loop, it is quite simple, neat and easy to understand.
package required: fs, express-fileupload

server.js :-

router.post('/upload', function(req, res){
    if(req.files){
        var file = req.files.filename;
        test(file);
    res.render('test');
}
} );

test function () :-

function test(file){
  // upload all
  if(file.length){
    for(var i =0; i < file.length; i++){
      fileUP(file[i]); 
    }
  }else{
    fileUP(file);
}

  // call fileUP() to upload 1 at once
  function fileUP(fyl){
    var filename = fyl.name;
    var tempPath = './temp'+filename;
    fyl.mv(tempPath, function(err){
    fs.readFile(tempPath, function(err, data){
    var params = {
      Bucket: 'BUCKET_NAME',
      Body: data,
      Key: Date.now()+filename
    };

    s3.upload(params, function (err, data) {
      if (data) {
        fs.unlink(tempPath, (err) => {
          if (err) {
            console.error(err)
            return
          }
          else{
            console.log("file removed from temp loaction");
          }
        });
        console.log("Uploaded in:", data.Location);
      }
    });
  });
  });
  }

}

Answer 3

Here's Node code I wrote to assemble the S3 objects from truncated lists.

var params = {
    Bucket: <yourbucket>,
    Prefix: <yourprefix>,
};

var s3DataContents = [];    // Single array of all combined S3 data.Contents

function s3Print() {
    if (program.al) {
        // --al: Print all objects
        console.log(JSON.stringify(s3DataContents, null, "    "));
    } else {
        // --b: Print key only, otherwise also print index 
        var i;
        for (i = 0; i < s3DataContents.length; i++) {
            var head = !program.b ? (i+1) + ': ' : '';
            console.log(head + s3DataContents[i].Key);
        }
    }
}

function s3ListObjects(params, cb) {
    s3.listObjects(params, function(err, data) {
        if (err) {
            console.log("listS3Objects Error:", err);
        } else {
            var contents = data.Contents;
            s3DataContents = s3DataContents.concat(contents);
            if (data.IsTruncated) {
                // Set Marker to last returned key
                params.Marker = contents[contents.length-1].Key;
                s3ListObjects(params, cb);
            } else {
                cb();
            }
        }
    });
}

s3ListObjects(params, s3Print);

Pay attention to listObject's documentation of NextMarker, which is NOT always present in the returned data object, so I don't use it at all in the above code ...

NextMarker — (String) When response is truncated (the IsTruncated element value in the response is true), you can use the key name in this field as marker in the subsequent request to get next set of objects. Amazon S3 lists objects in alphabetical order Note: This element is returned only if you have delimiter request parameter specified. If response does not include the NextMarker and it is truncated, you can use the value of the last Key in the response as the marker in the subsequent request to get the next set of object keys.

The entire program has now been pushed to https://github.com/kenklin/s3list.