How to Increase Upload Speed for GoogleCloudStorage in nodejs
Uploading Files to GCP : Google Cloud Storage In Nodejs
const Storage = require('@google-cloud/storage');
const config = require('./gcloudConfig');
const Multer = require('multer');
const CLOUD_BUCKET = config.get('CLOUD_BUCKET');
const storage = Storage({
projectId: config.get('GCLOUD_PROJECT')
});
const bucket = storage.bucket(CLOUD_BUCKET);
// Returns the public, anonymously accessable URL to a given Cloud Storage
// object.
// The object's ACL has to be set to public read.
// [START public_url]
function getPublicUrl (filename) {
return `https://storage.googleapis.com/${CLOUD_BUCKET}/${filename}`;
}
// Express middleware that will automatically pass uploads to Cloud Storage.
// req.file is processed and will have two new properties:
// * ``cloudStorageObject`` the object name in cloud storage.
// * ``cloudStoragePublicUrl`` the public url to the object.
// [START process]
function sendUploadToGCS (req, res, next) {
if (!req.files) {
return next();
}
const data = req.body;
if (req.files.length === 0) {
next();
}
data.accountNumber = '';
for (let index = 0; index < req.files.length; index++) {
const gcsname = `${data.accountNumber}/${Date.now()}${req.files[index].originalname}`;
const file = bucket.file(gcsname);
const stream = file.createWriteStream({
metadata: {
contentType: req.files[index].mimetype
},
resumable: false
});
stream.on('error', (err) => {
req.files[index].cloudStorageError = err;
next(err);
});
// console.log(' upload in progress for file ', index);
stream.on('finish', () => {
req.files[index].cloudStorageObject = gcsname;
file.makePublic().then(() => {
req.files[index].cloudStoragePublicUrl = getPublicUrl(gcsname);
console.log(' upload completed, public url is ', req.files[index].cloudStoragePublicUrl,
' index ', index, ' total count ', req.files.length);
if (index === (req.files.length - 1)) {
next();
}
});
});
stream.end(req.files[index].buffer);
}
}
// [END process]
// Multer handles parsing multipart/form-data requests.
// This instance is configured to store images in memory.
// This makes it straightforward to upload to Cloud Storage.
// [START multer]
const multer = Multer({
storage: Multer.MemoryStorage,
limits: {
fileSize: 5 * 1024 * 1024 // no larger than 5mb
}
});
// [END multer]
module.exports = {
getPublicUrl,
sendUploadToGCS,
multer
};
The issue is the upload speed is very slow, finding a way to upload faster to google cloud storage