我需要一些帮助来正确地构建代码,使用S3存储桶和Lambda函数处理一些文本文件。
我想使用一个Lambda函数,当在S3存储桶中创建新对象时触发该函数,以读取文件并提取某些数据,然后将这些数据写入到放置在另一个S3存储桶中的文件中。
到目前为止,我的函数已经可以很好地从一个S3存储桶复制文件到另一个S3存储桶,但我还不太清楚如何添加一个函数来处理文件并将结果写入最终的S3目标。
这些文件是简单的文本文件,我需要从每行中提取数据。
以下是我目前正在使用的Node.js代码,其中添加了一个附加函数来处理文件 - 请参见带有“?”的注释,那里是我需要帮助的地方。
// dependencies
var async = require('async');
var AWS = require('aws-sdk');
var util = require('util');
// get reference to S3 client
var s3 = new AWS.S3();
exports.handler = function(event, context) {
// Read options from the event.
console.log("Reading options from event:\n", util.inspect(event, {depth: 5}));
var srcBucket = event.Records[0].s3.bucket.name;
// Object key may have spaces or unicode non-ASCII characters.
var srcKey =
decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " "));
var dstBucket = "inputBucket";
var dstKey = srcKey + ".txt";
// Sanity check: validate that source and destination are different buckets.
if (srcBucket == dstBucket) {
console.error("Destination bucket must not match source bucket.");
return;
}
// Infer the file type.
var typeMatch = srcKey.match(/\.([^.]*)$/);
if (!typeMatch) {
console.error('unable to infer file type for key ' + srcKey);
return;
}
var imageType = typeMatch[1];
if (imageType != "txt") {
console.log('skipping non-image ' + srcKey);
return;
}
// Download the image from S3, transform, and upload to a different S3 bucket.
async.waterfall([
function download(next) {
// Download the file from S3 into a buffer.
s3.getObject({
Bucket: srcBucket,
Key: srcKey
},
next);
},
function transform(response, next) {
// Read the file we have just downloaded
// ? response.Body ?
var rl = require('readline').createInterface({
input: require('fs').createReadStream('file.in')
});
// Process each line here writing the result to an output buffer?
rl.on('line', function (line) {
console.log('Line from file:', line);
//Do something with the line...
//Create some output string 'outputline'
//Write 'outputline' to an output buffer 'outbuff'
// ??
});
// Now pass the output buffer to the next function
// so it can be uploaded to another S3 bucket
// ??
next;
}
function upload(response, next) {
// Stream the file to a different S3 bucket.
s3.putObject({
Bucket: dstBucket,
Key: dstKey,
Body: response.Body,
ContentType: response.contentType
},
next);
}
], function (err) {
if (err) {
console.error(
'Unable to process ' + srcBucket + '/' + srcKey +
' and upload to ' + dstBucket + '/' + dstKey +
' due to an error: ' + err
);
} else {
console.log(
'Successfully processed ' + srcBucket + '/' + srcKey +
' and uploaded to ' + dstBucket + '/' + dstKey
);
}
context.done();
}
);
};