How to Increase Upload Speed for GoogleCloudStorage in nodejs
Uploading Files to GCP : Google Cloud Storage In Nodejs
const Storage = require('@google-cloud/storage'); const config = require('./gcloudConfig'); const Multer = require('multer'); const CLOUD_BUCKET = config.get('CLOUD_BUCKET'); const storage = Storage({ projectId: config.get('GCLOUD_PROJECT') }); const bucket = storage.bucket(CLOUD_BUCKET); // Returns the public, anonymously accessable URL to a given Cloud Storage // object. // The object's ACL has to be set to public read. // [START public_url] function getPublicUrl (filename) { return `https://storage.googleapis.com/${CLOUD_BUCKET}/${filename}`; } // Express middleware that will automatically pass uploads to Cloud Storage. // req.file is processed and will have two new properties: // * ``cloudStorageObject`` the object name in cloud storage. // * ``cloudStoragePublicUrl`` the public url to the object. // [START process] function sendUploadToGCS (req, res, next) { if (!req.files) { return next(); } const data = req.body; if (req.files.length === 0) { next(); } data.accountNumber = ''; for (let index = 0; index < req.files.length; index++) { const gcsname = `${data.accountNumber}/${Date.now()}${req.files[index].originalname}`; const file = bucket.file(gcsname); const stream = file.createWriteStream({ metadata: { contentType: req.files[index].mimetype }, resumable: false }); stream.on('error', (err) => { req.files[index].cloudStorageError = err; next(err); }); // console.log(' upload in progress for file ', index); stream.on('finish', () => { req.files[index].cloudStorageObject = gcsname; file.makePublic().then(() => { req.files[index].cloudStoragePublicUrl = getPublicUrl(gcsname); console.log(' upload completed, public url is ', req.files[index].cloudStoragePublicUrl, ' index ', index, ' total count ', req.files.length); if (index === (req.files.length - 1)) { next(); } }); }); stream.end(req.files[index].buffer); } } // [END process] // Multer handles parsing multipart/form-data requests. // This instance is configured to store images in memory. // This makes it straightforward to upload to Cloud Storage. // [START multer] const multer = Multer({ storage: Multer.MemoryStorage, limits: { fileSize: 5 * 1024 * 1024 // no larger than 5mb } }); // [END multer] module.exports = { getPublicUrl, sendUploadToGCS, multer };
The issue is the upload speed is very slow, finding a way to upload faster to google cloud storage