大文件上传到node.js中的Azure blob处理
大文件上传到node.js中的Azure blob处理
提问于 2019-07-30 06:44:28
我的目标是将一个大文件(任何文件类型)上传到Azure blob中。
我想遵循以下两种方法。
1. Convert file into stream and upload into Azure blob
2. Convert file into block streams and commit all the blocks to Azure blob in parallel我正在使用Node.js Azure blob服务API的
方法1:我使用ReadStream将我的文件转换为fs.createReadStream(fileName, 'utf-8')。并使用blobService.createBlockBlobFromStream()将该流上传到Azure blob。下面是这种方法的示例代码
var blobName = "azure-blob-name.doc";
var fileSize = fs.statSync(fileName).size;
var readableStream = fs.createReadStream(fileName, 'utf-8');
blobService.createBlockBlobFromStream(containerName, blobName, readableStream, fileSize, function (error, result, response) {
if(error) console.log(error);
console.log({message : "blob uploaded"});
});方法2:我希望通过使用ReadStream并并行提交所有块，将blobService.createBlockFromStream()转换为蓝色blob块。我怎样才能达到这个目的呢？
下面是我想要做的示例代码，但最后抛出了一个错误
var fileName = "local-file-path/filename.doc";
var blobName = "azure-blob-name.doc";
var fileSize = fs.statSync(fileName).size;
var readableStream = fs.createReadStream(fileName, 'utf-8');
var bl=1;
var blockIds = [];
readableStream.on("data", function (chunk) {
var blockId = 'block'+bl;
bl++;
blobService.createBlockFromStream(blockId, containerName, blobName, chunk, chunk.length, function(error, response){
if(error) console.log(error);
blockIds.push(blockId);
console.log({message : "block created"});
});
});下面是我正在犯的错误
D:\main-workspace\file-management\node_modules\azure-storage\lib\common\services\storageserviceclient.js:522
body.outputStream.on('open', function () {
^
TypeError: body.outputStream.on is not a function- 方法1工作良好，但上传时间过长。(统计:上传40 MB文件需要80秒)。
- 我的第二个方法是提高性能的正确方法吗？如果是的话，我怎样才能达到这个目的呢？
是否有更好的方法来提高性能？在这方面有什么建议吗？
除了Azure blob服务API之外，我们还可以这样做吗？我的目标是在更短的时间内上传任何类型的大文件。例如，40 MB文件必须在20秒内上传。
请把我开向正确的方向。谢谢你提前给我答复。
编辑-更新代码
@Gaurav
下面是我更新的代码。但是我的readableStream.on("end", function(){})从来没有被调用来提交块。在我的Azure帐户中，文件正在创建，但要用0字节
const stream = require('stream');
var fileName = "local-file-path/filename.doc";
var blobName = "azure-blob-name.doc";
var fileSize = fs.statSync(blobName).size;
var customBlockSize = 1200; //1.2 MB CAP
var NoOfBlocks = Math.ceil(fileSize/customBlockSize);
var blockIdLength = NoOfBlocks.toString().length;
var readableStream = fs.createReadStream(fileName, 'utf-8');
var bl=1;
var blockIds = [];
readableStream.on("data", function (chunk) {
var blockId = 'block'+ blockGuId(bl, blockIdLength);//Block ids must be of same length
bl++;
const bufferStream = new stream.PassThrough({
highWaterMark: chunk.length
});
bufferStream.end(chunk);
blobService.createBlockFromStream(blockId, containerName, blobName, bufferStream, chunk.length, function(error, response){
if(error) console.log(error);
blockIds.push(blockId);
console.log({message : "block ("+blockId+") created"});
});
});
readableStream.on("end", function(){
console.log('readableStream.on "end"');
blobService.commitBlocks(containerName, blobName, blockIds, function(error, result){
if(error) console.log(error);
console.log({message : "all blocks uploaded"});
var hrend = process.hrtime(hrstart)
console.info('Execution time (sec ms): %ds %dms', hrend[0], hrend[1] / 1000000)
});
});
blockGuId(number, blockIdLength)
{
return Array(Math.max(blockIdLength - String(number).length + 1, 0)).join(0) + number;
}下面是套接字错误
{ Error: socket hang up at createHangUpError (_http_client.js:323:15) at TLSSocket.socketCloseListener (_http_client.js:364:25) at TLSSocket.emit (events.js:187:15) at _handle.close (net.js:610:12) at TCP.done (_tls_wrap.js:386:7) code: 'ECONNRESET' }我在这里错过了什么？为什么在创建块时出现套接字错误？
回答 1
Stack Overflow用户
发布于 2019-07-30 07:44:25
您能试一下以下代码吗？
const stream = require('stream');
var fileName = "local-file-path/filename.doc";
var blobName = "azure-blob-name.doc";
var fileSize = fs.statSync(fileName).size;
var readableStream = fs.createReadStream(fileName, 'utf-8');
var bl=1;
var blockIds = [];
readableStream.on("data", function (chunk) {
//var blockId = 'block'+bl;
bl++;
var blockId = UseGuidHere();//Block ids must be of same length
const bufferStream = new stream.PassThrough({
highWaterMark: chunk.length
});
bufferStream.end(chunk);
blobService.createBlockFromStream(blockId, containerName, blobName, bufferStream, chunk.length, function(error, response){
if(error) console.log(error);
blockIds.push(blockId);
console.log({message : "block created"});
});
});正如代码中提到的，请为您的块id指定一个GUID，因为块id必须具有相同的长度。使用当前的逻辑，在9块之后，块id的长度将改变，这将导致上传过程中的失败。
页面原文内容由Stack Overflow提供。腾讯云小微IT领域专用引擎提供翻译支持
原文链接：
https://stackoverflow.com/questions/57265764
复制相关文章
相似问题
腾讯云开发者
