将图片从URL抓取到S3
将图片从URL抓取到S3
提问于 2019-10-24 03:57:54
其目标是让AWS lambda每天将特定的jpg从URL复制到S3存储桶中。这看起来像是一个超级简单的lambda函数，但它就是不起作用。我将此设置作为本地NPM项目，然后在AWS lambda控制台中上传压缩文件。下面的代码运行时没有错误，但没有复制镜像。任何帮助都将不胜感激。
"use strict";
const AWS = require("aws-sdk");
const Jimp = require("jimp");
const s3 = new AWS.S3();
const imageType = "image/jpeg";
const bucket = 'mybucket';
exports.handler = (event, context, callback) => {
let objectKey = 'myimage.jpg';
Jimp.read('sampleURL.com/image.jpg')
.then(image => {
s3.putObject({
Bucket: bucket,
Key: objectKey,
Body: image,
ContentType: imageType
})
})
.catch(err => {
// Handle an exception.
});
};aws cloudwatch日志
2019-10-24T12:48:23.105Z bac7d80e-5544-4ea5-ae12-478281338389 INFO { Error: Could not find MIME for Buffer <null>
at Jimp.parseBitmap (/var/task/node_modules/@jimp/core/dist/utils/image-bitmap.js:120:15)
at Jimp.parseBitmap (/var/task/node_modules/@jimp/core/dist/index.js:506:32)
at /var/task/node_modules/@jimp/core/dist/index.js:448:15
at /var/task/node_modules/@jimp/core/dist/index.js:176:14
at /var/task/node_modules/@jimp/core/dist/request.js:66:9
at IncomingMessage.<anonymous> (/var/task/node_modules/phin/lib/phin.compiled.js:1:2100)
at IncomingMessage.emit (events.js:203:15)
at IncomingMessage.EventEmitter.emit (domain.js:448:20)
at endReadableNT (_stream_readable.js:1145:12)
at process._tickCallback (internal/process/next_tick.js:63:19) methodName: 'constructor' }
END RequestId: bac7d80e-5544-4ea5-ae12-478281338389
REPORT RequestId: bac7d80e-5544-4ea5-ae12-478281338389 Duration: 612.63 ms Billed Duration: 700 ms Memory Size: 128 MB Max Memory Used: 97 MB Init Duration: 557.69 ms 回答 3
Stack Overflow用户
发布于 2019-10-24 05:53:20
在这些情况下，一个常见的问题是权限错误。要允许AWS Lambda在S3中执行PutObject，您需要在Lambda执行角色中设置此类权限。
下面是一个允许Lambda在S3中做任何事情的策略示例：
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:ListAllMyBuckets",
"s3:GetBucketLocation"
],
"Resource": "*"
},
{
"Effect": "Allow",
"Action": "s3:*",
"Resource": [
"arn:aws:s3:::ExampleBucketName",
"arn:aws:s3:::ExampleBucketName/*"
]
}
]
}出于安全考虑，重要的是将操作限制为您实际需要的操作，例如PutObject，并且您可以获得有关this link的更多信息。
Stack Overflow用户
发布于 2019-10-24 10:07:37
下面是一个如何将文件从HTTP URL流式传输到S3的示例。它使用promises/async/await而不是回调，而且它省去了Jimp包，我对它知之甚少，取而代之的是更传统的fetch应用编程接口：
注意:如果在上传到S3时没有显式地提供content-type，那么它将被设置为application/octet-stream，这在客户端下载对象时会有问题。因此，此代码首先确定文件的内容类型，并在流式传输到S3时对其进行设置。
const AWS = require('aws-sdk');
const fetch = require('node-fetch');
const stream = require('stream');
const s3 = new AWS.S3();
const uploadStream = ({ Bucket, Key, ContentType }) => {
const pass = new stream.PassThrough();
return {
writeStream: pass,
promise: s3.upload({ Bucket, Key, ContentType, Body: pass }).promise(),
};
}
const uploadFetch = async ({ url, Bucket, Key, ContentType }) => {
const response = await fetch(url);
const { writeStream, promise } = uploadStream({Bucket, Key, ContentType});
response.body.pipe(writeStream);
return promise;
}
exports.handler = async (_event, _context) => {
const source_jpeg = {
Key: 'audi.jpeg',
Bucket: 'mybucket',
url: 'https://upload.wikimedia.org/wikipedia/commons/0/08/Audi_A3_2015.jpeg',
};
// HEAD the source image to get content type
const rc_head = await fetch(source_jpeg.url, {method: 'HEAD'});
const content_type = rc_head.headers.get('content-type');
console.log('head:', rc_head.status, rc_head.statusText, content_type);
try {
// GET the source image and stream it to S3
const parms = {...source_jpeg, ContentType: content_type};
const rc_upload = await uploadFetch(parms);
console.log('get/upload jpeg:', rc_upload);
} catch(e) {
console.log(e);
}
};另外，确保您的Lambda函数配置了合理的超时(默认超时为3秒)。
Stack Overflow用户
发布于 2019-10-26 00:04:56
如果这对其他任何人都有帮助，则需要将图像写入缓冲区。下面这行代码修复了这个问题：
const buffer = await image.getBufferAsync(imageType);然后将缓冲区用于S3主体参数。所以完整的脚本是：
"use strict";
const AWS = require("aws-sdk");
const Jimp = require("jimp");
const s3 = new AWS.S3();
const imageType = "image/jpeg";
const bucket = 'bucketxzy';
exports.handler = async (event, context) => {
let objectKey = 'sampleimage.jpeg';
const image = await Jimp.read('https://www.sampleurl.com/sampleimage.jpg/');
const buffer = await image.getBufferAsync(imageType);
return s3.putObject({
Bucket: bucket,
Key: objectKey,
Body: buffer,
ContentType: imageType
}).promise();
};页面原文内容由Stack Overflow提供。腾讯云小微IT领域专用引擎提供翻译支持
原文链接：
https://stackoverflow.com/questions/58530146
复制相关文章
相似问题
腾讯云开发者
