From 50f37fd1a291aded3e9e2f1499754396f54f5e2c Mon Sep 17 00:00:00 2001 From: Jack Foltz Date: Sun, 29 Jul 2018 20:08:52 -0400 Subject: [PATCH] Rewrite upload code to phase out Multer, allowing for more granular control --- app/routes/upload.js | 79 ++---------------------------- app/util/upload/disk.js | 28 +++++++++++ app/util/upload/id.js | 23 +++++++++ app/util/upload/multipart.js | 114 +++++++++++++++++++++++++++++++++++++++++++ app/util/upload/stats.js | 13 +++++ package-lock.json | 29 +++-------- package.json | 3 +- 7 files changed, 192 insertions(+), 97 deletions(-) create mode 100644 app/util/upload/disk.js create mode 100644 app/util/upload/id.js create mode 100644 app/util/upload/multipart.js create mode 100644 app/util/upload/stats.js diff --git a/app/routes/upload.js b/app/routes/upload.js index e96666f..fdaefb9 100644 --- a/app/routes/upload.js +++ b/app/routes/upload.js @@ -2,86 +2,17 @@ const express = require('express'); const router = express.Router(); const config = require('config'); -const fsPromises = require('fs').promises; - const ModelPath = '../models/'; -const User = require(ModelPath + 'User.js'); const Upload = require(ModelPath + 'Upload.js'); -const Key = require(ModelPath + 'Key.js'); -const verifyScope = require('../util/verifyScope.js'); - -const multer = require('multer'); -const fileUpload = multer({dest: config.get('Upload.path')}).single('file'); +const uploadMultipart = require('../util/upload/multipart'); +const updateStats = require('../util/upload/stats'); const wrap = require('../util/wrap.js'); -const generatedIdExists = async id => - await Upload.countDocuments({id: id}) === 1; - -const generateId = async() => { - const charset = config.get('Upload.charset'); - const len = config.get('Upload.idLength'); - - const id = [...Array(len)] - .map(() => charset.charAt(Math.floor(Math.random() * charset.length))) - .join(''); - - return await generatedIdExists(id) - ? generateId() - : id; -}; - -const updateStats = async req => - Promise.all([ - User.updateOne({username: req.username}, {$inc: {uploadCount: 1, uploadSize: req.file.size}}), - req.key - ? Key.updateOne({key: req.key}, {$inc: {uploadCount: 1, uploadSize: req.file.size}}) - : Promise.resolve() - ]); - - -router.post('/', fileUpload, wrap(async(req, res) => { - // We need to authenticate in place because the form data needs to be processed by multer first - const deleteAndError = async (code, message) => { - if (req.file) - await fsPromises.unlink(req.file.path); - res.status(code).json({message: message}); - }; - - if (req.isAuthenticated()) { - if (verifyScope(req.session.passport.scope, 'file.upload')) { - req.username = req.session.passport.user; - req.displayname = req.session.passport.displayname; - req.scope = req.session.passport.scope; - req.key = null; - } else { - return await deleteAndError(403, 'Forbidden.'); - } - } else if (req.body.key) { - const key = await Key.findOne({key: req.body.key}); - if (verifyScope(key.scope, 'file.upload')) { - req.username = key.issuer; - req.displayname = key.issuer; - req.scope = key.scope; - req.key = key.key; - } else { - return await deleteAndError(403, 'Forbidden.'); - } - } else { - return await deleteAndError(401, 'Unauthorized.'); - } - - if (!req.file) - return res.status(400).json({message: 'No file specified.'}); - - if (req.file.size > config.get('Upload.maxSize')) { - await fsPromises.unlink(req.file.path); - return res.status(413).json({message: 'File too large.'}); - } - +router.post('/', uploadMultipart, wrap(async (req, res) => { const upload = { - id: await generateId(), + id: req.file.name, uploader: req.username, uploaderKey: req.key, date: Date.now(), @@ -95,7 +26,7 @@ router.post('/', fileUpload, wrap(async(req, res) => { res.status(200).json({ message: 'File uploaded.', - id: upload.id, + id: req.file.name, url: config.get('Server.hostname') + '/v/' + upload.id }); })); diff --git a/app/util/upload/disk.js b/app/util/upload/disk.js new file mode 100644 index 0000000..c68d47c --- /dev/null +++ b/app/util/upload/disk.js @@ -0,0 +1,28 @@ +const fs = require('fs'); +const fsPromises = fs.promises; + +const mkdir = path => new Promise((resolve, reject) => { + fsPromises.mkdir(path) + .then(resolve) + .catch(err => { + if (err.code === 'EEXIST') + resolve(); + else + reject(err); + }); +}); + +const write = (path, stream) => new Promise((resolve, reject) => { + const outStream = fs.createWriteStream(path); + stream.pipe(outStream); + outStream.on('error', reject); + outStream.on('close', () => resolve(outStream.bytesWritten)); + outStream.on('finish', () => resolve(outStream.bytesWritten)); +}); + +const remove = path => + fsPromises.unlink(path); + +exports.mkdir = mkdir; +exports.write = write; +exports.remove = remove; \ No newline at end of file diff --git a/app/util/upload/id.js b/app/util/upload/id.js new file mode 100644 index 0000000..c7f1981 --- /dev/null +++ b/app/util/upload/id.js @@ -0,0 +1,23 @@ +const config = require('config'); + +const ModelPath = '../../models/'; +const Upload = require(ModelPath + 'Upload.js'); + +const exists = async id => + await Upload.countDocuments({id: id}) === 1; + +const generate = async () => { + const charset = config.get('Upload.charset'); + const len = config.get('Upload.idLength'); + + const id = [...Array(len)] + .map(() => charset.charAt(Math.floor(Math.random() * charset.length))) + .join(''); + + return await exists(id) + ? generate() + : id; +}; + +exports.generate = generate; +exports.exists = exists; \ No newline at end of file diff --git a/app/util/upload/multipart.js b/app/util/upload/multipart.js new file mode 100644 index 0000000..7bf06eb --- /dev/null +++ b/app/util/upload/multipart.js @@ -0,0 +1,114 @@ +const Busboy = require('busboy'); +const is = require('type-is'); +const config = require('config'); + +const wrap = require('../wrap'); +const auth = require('../auth'); +const disk = require('./disk'); +const identifier = require('./id'); + +const uploadMultipart = wrap(async (req, res, next) => { + if (!is(req, ['multipart'])) + return res.status(400).json({message: 'Bad request.'}); + + // Store whether the user has authenticated, because an api key might be included with the form later + let authStatus = { + authenticated: false, + permission: false + }; + // If not authenticated with a session, we'll have to wait for key authentication from the multipart form data + await auth.checkSession(req, 'file.upload', authStatus); + + // Function to call once the file is sent or an error is encountered + let isDone = false; + const done = async (err, data) => { + if (isDone) return; + isDone = true; + + req.unpipe(busboy); + busboy.removeAllListeners(); + req.on('readable', req.read.bind(req)); + + if (err) { + if (data.path) + await disk.remove(data.path); + + if (err === 'LIMIT_FILE_SIZE') + return res.status(413).json({message: 'File too large.'}); + else { + console.log(err.stack); + return res.status(500).json({message: 'Internal server error.'}); + } + } else { + req.file = data.file; + next(); + } + }; + + // Create the busboy object to parse the multipart data + const busboy = new Busboy({ + headers: req.headers, + limits: { + fileSize: config.get('Upload.maxSize') + } + }); + + req.body = {}; + busboy.on('field', (fieldName, value) => { + req.body[fieldName] = value; + }); + + let fileCount = 0; + let file; + busboy.on('file', async (fieldName, stream, name, encoding, mime) => { + // Only process one file + fileCount++; + if (fileCount > 1) + return res.status(400).json({message: 'Bad request.'}); + + // If a key was encountered and we are not authenticated, try to authenticate with it before the final check + if (req.body.key && !authStatus.authenticated) + await auth.checkKey(req, 'file.upload', authStatus); + + // Finally, check if we have auth before preceeding, keys should have been processed by now + if (!authStatus.authenticated) + return res.status(401).json({message: 'Unauthorized.'}); + if (!authStatus.permission) + return res.status(403).json({message: 'Forbidden.'}); + + // Don't attach to the files object if there is no file + if (!name) return stream.resume(); + + // Generate an ID for the file, saving it with that filename + const path = config.get('Upload.path'); + const newName = await identifier.generate(); + const finalPath = path + '/' + newName; + + // Set the file attributes + file = { + originalName: name, + name: newName, + path: finalPath, + encoding: encoding, + mime: mime + }; + + // Ensure the output directory exists + await disk.mkdir(path); + + // Handle errors + stream.on('error', err => done(err, {path: finalPath})); + stream.on('limit', () => done('LIMIT_FILE_SIZE', {path: finalPath})); + + file.size = await disk.write(finalPath, stream); + + await done(null, {file: file}); + }); + + busboy.on('error', err => done(err)); + busboy.on('finished', () => done(null, {file: file})); + + req.pipe(busboy); +}); + +module.exports = uploadMultipart; \ No newline at end of file diff --git a/app/util/upload/stats.js b/app/util/upload/stats.js new file mode 100644 index 0000000..9b2aa4d --- /dev/null +++ b/app/util/upload/stats.js @@ -0,0 +1,13 @@ +const ModelPath = '../../models/'; +const User = require(ModelPath + 'User.js'); +const Key = require(ModelPath + 'Key.js'); + +const updateStats = req => + Promise.all([ + User.updateOne({username: req.username}, {$inc: {uploadCount: 1, uploadSize: req.file.size}}), + req.key + ? Key.updateOne({key: req.key}, {$inc: {uploadCount: 1, uploadSize: req.file.size}}) + : Promise.resolve() + ]); + +module.exports = updateStats; \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index c345d64..325f9a5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -152,11 +152,6 @@ "buffer-equal": "^1.0.0" } }, - "append-field": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/append-field/-/append-field-0.1.0.tgz", - "integrity": "sha1-bdxY+gg8e8VF08WZWygwzCNm1Eo=" - }, "archy": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz", @@ -1249,6 +1244,7 @@ "version": "1.6.0", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.0.tgz", "integrity": "sha1-CqxmL9Ur54lk1VMvaUeE5wEQrPc=", + "dev": true, "requires": { "inherits": "^2.0.3", "readable-stream": "^2.2.2", @@ -1258,12 +1254,14 @@ "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "dev": true }, "readable-stream": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.3.tgz", "integrity": "sha512-m+qzzcn7KUxEmd1gMbchF+Y2eIUbieUaxkWtptyHywrX0rE8QEYqPC07Vuy4Wm32/xE16NcdBctb8S0Xe/5IeQ==", + "dev": true, "requires": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -1278,6 +1276,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz", "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==", + "dev": true, "requires": { "safe-buffer": "~5.1.0" } @@ -5864,21 +5863,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, - "multer": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/multer/-/multer-1.3.1.tgz", - "integrity": "sha512-JHdEoxkA/5NgZRo91RNn4UT+HdcJV9XUo01DTkKC7vo1erNIngtuaw9Y0WI8RdTlyi+wMIbunflhghzVLuGJyw==", - "requires": { - "append-field": "^0.1.0", - "busboy": "^0.2.11", - "concat-stream": "^1.5.2", - "mkdirp": "^0.5.1", - "object-assign": "^3.0.0", - "on-finished": "^2.3.0", - "type-is": "^1.6.4", - "xtend": "^4.0.0" - } - }, "multipipe": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/multipipe/-/multipipe-0.1.2.tgz", @@ -11910,7 +11894,8 @@ "typedarray": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" + "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=", + "dev": true }, "uglify-js": { "version": "3.4.5", diff --git a/package.json b/package.json index 94faafb..fa3e8f0 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "angular-ui-router": "^1.0.19", "async": "^2.6.1", "body-parser": "^1.18.3", + "busboy": "^0.2.14", "config": "^1.31.0", "connect-mongo": "^2.0.1", "express": "^4.16.3", @@ -17,12 +18,12 @@ "method-override": "latest", "mongoose": "^5.2.5", "morgan": "^1.9.0", - "multer": "^1.3.1", "ng-file-upload": "^12.2.13", "ngclipboard": "^2.0.0", "passport": "^0.4.0", "passport-local": "^1.0.0", "passport-local-mongoose": "^5.0.1", + "type-is": "^1.6.16", "vinyl-source-stream": "^2.0.0" }, "description": "A simple file sharing website.",