Rewrite upload code to phase out Multer, allowing for more granular control

app/routes/upload.js

@@ -2,86 +2,17 @@ const express = require('express');
 const router = express.Router();
 const config = require('config');
 
-const fsPromises = require('fs').promises;
-
 const ModelPath = '../models/';
-const User = require(ModelPath + 'User.js');
 const Upload = require(ModelPath + 'Upload.js');
-const Key = require(ModelPath + 'Key.js');
 
-const verifyScope = require('../util/verifyScope.js');
-
-const multer = require('multer');
-const fileUpload = multer({dest: config.get('Upload.path')}).single('file');
+const uploadMultipart = require('../util/upload/multipart');
+const updateStats = require('../util/upload/stats');
 
 const wrap = require('../util/wrap.js');
 
-const generatedIdExists = async id =>
-    await Upload.countDocuments({id: id}) === 1;
-
-const generateId = async() => {
-    const charset = config.get('Upload.charset');
-    const len = config.get('Upload.idLength');
-
-    const id = [...Array(len)]
-        .map(() => charset.charAt(Math.floor(Math.random() * charset.length)))
-        .join('');
-
-    return await generatedIdExists(id)
-        ? generateId()
-        : id;
-};
-
-const updateStats = async req =>
-    Promise.all([
-        User.updateOne({username: req.username}, {$inc: {uploadCount: 1, uploadSize: req.file.size}}),
-        req.key
-            ? Key.updateOne({key: req.key}, {$inc: {uploadCount: 1, uploadSize: req.file.size}})
-            : Promise.resolve()
-    ]);
-
-
-router.post('/', fileUpload, wrap(async(req, res) => {
-    // We need to authenticate in place because the form data needs to be processed by multer first
-    const deleteAndError = async (code, message) => {
-        if (req.file)
-            await fsPromises.unlink(req.file.path);
-        res.status(code).json({message: message});
-    };
-
-    if (req.isAuthenticated()) {
-        if (verifyScope(req.session.passport.scope, 'file.upload')) {
-            req.username = req.session.passport.user;
-            req.displayname = req.session.passport.displayname;
-            req.scope = req.session.passport.scope;
-            req.key = null;
-        } else {
-            return await deleteAndError(403, 'Forbidden.');
-        }
-    } else if (req.body.key) {
-        const key = await Key.findOne({key: req.body.key});
-        if (verifyScope(key.scope, 'file.upload')) {
-            req.username = key.issuer;
-            req.displayname = key.issuer;
-            req.scope = key.scope;
-            req.key = key.key;
-        } else {
-            return await deleteAndError(403, 'Forbidden.');
-        }
-    } else {
-        return await deleteAndError(401, 'Unauthorized.');
-    }
-
-    if (!req.file)
-        return res.status(400).json({message: 'No file specified.'});
-
-    if (req.file.size > config.get('Upload.maxSize')) {
-        await fsPromises.unlink(req.file.path);
-        return res.status(413).json({message: 'File too large.'});
-    }
-
+router.post('/', uploadMultipart, wrap(async (req, res) => {
     const upload = {
-        id: await generateId(),
+        id: req.file.name,
         uploader: req.username,
         uploaderKey: req.key,
         date: Date.now(),
@@ -95,7 +26,7 @@ router.post('/', fileUpload, wrap(async(req, res) => {
 
     res.status(200).json({
         message: 'File uploaded.',
-        id: upload.id,
+        id: req.file.name,
         url: config.get('Server.hostname') + '/v/' + upload.id
     });
 }));

app/util/upload/disk.js

@@ -0,0 +1,28 @@
+const fs = require('fs');
+const fsPromises = fs.promises;
+
+const mkdir = path => new Promise((resolve, reject) => {
+    fsPromises.mkdir(path)
+        .then(resolve)
+        .catch(err => {
+            if (err.code === 'EEXIST')
+                resolve();
+            else
+                reject(err);
+        });
+});
+
+const write = (path, stream) => new Promise((resolve, reject) => {
+    const outStream = fs.createWriteStream(path);
+    stream.pipe(outStream);
+    outStream.on('error', reject);
+    outStream.on('close', () => resolve(outStream.bytesWritten));
+    outStream.on('finish', () => resolve(outStream.bytesWritten));
+});
+
+const remove = path =>
+    fsPromises.unlink(path);
+
+exports.mkdir = mkdir;
+exports.write = write;
+exports.remove = remove;

app/util/upload/id.js

@@ -0,0 +1,23 @@
+const config = require('config');
+
+const ModelPath = '../../models/';
+const Upload = require(ModelPath + 'Upload.js');
+
+const exists = async id =>
+    await Upload.countDocuments({id: id}) === 1;
+
+const generate = async () => {
+    const charset = config.get('Upload.charset');
+    const len = config.get('Upload.idLength');
+
+    const id = [...Array(len)]
+        .map(() => charset.charAt(Math.floor(Math.random() * charset.length)))
+        .join('');
+
+    return await exists(id)
+        ? generate()
+        : id;
+};
+
+exports.generate = generate;
+exports.exists = exists;

app/util/upload/multipart.js

@@ -0,0 +1,114 @@
+const Busboy = require('busboy');
+const is = require('type-is');
+const config = require('config');
+
+const wrap = require('../wrap');
+const auth = require('../auth');
+const disk = require('./disk');
+const identifier = require('./id');
+
+const uploadMultipart = wrap(async (req, res, next) => {
+    if (!is(req, ['multipart']))
+        return res.status(400).json({message: 'Bad request.'});
+
+    // Store whether the user has authenticated, because an api key might be included with the form later
+    let authStatus = {
+        authenticated: false,
+        permission: false
+    };
+    // If not authenticated with a session, we'll have to wait for key authentication from the multipart form data
+    await auth.checkSession(req, 'file.upload', authStatus);
+
+    // Function to call once the file is sent or an error is encountered
+    let isDone = false;
+    const done = async (err, data) => {
+        if (isDone) return;
+        isDone = true;
+
+        req.unpipe(busboy);
+        busboy.removeAllListeners();
+        req.on('readable', req.read.bind(req));
+
+        if (err) {
+            if (data.path)
+                await disk.remove(data.path);
+
+            if (err === 'LIMIT_FILE_SIZE')
+                return res.status(413).json({message: 'File too large.'});
+            else {
+                console.log(err.stack);
+                return res.status(500).json({message: 'Internal server error.'});
+            }
+        } else {
+            req.file = data.file;
+            next();
+        }
+    };
+
+    // Create the busboy object to parse the multipart data
+    const busboy = new Busboy({
+        headers: req.headers,
+        limits: {
+            fileSize: config.get('Upload.maxSize')
+        }
+    });
+
+    req.body = {};
+    busboy.on('field', (fieldName, value) => {
+        req.body[fieldName] = value;
+    });
+
+    let fileCount = 0;
+    let file;
+    busboy.on('file', async (fieldName, stream, name, encoding, mime) => {
+        // Only process one file
+        fileCount++;
+        if (fileCount > 1)
+            return res.status(400).json({message: 'Bad request.'});
+
+        // If a key was encountered and we are not authenticated, try to authenticate with it before the final check
+        if (req.body.key && !authStatus.authenticated)
+            await auth.checkKey(req, 'file.upload', authStatus);
+
+        // Finally, check if we have auth before preceeding, keys should have been processed by now
+        if (!authStatus.authenticated)
+            return res.status(401).json({message: 'Unauthorized.'});
+        if (!authStatus.permission)
+            return res.status(403).json({message: 'Forbidden.'});
+
+        // Don't attach to the files object if there is no file
+        if (!name) return stream.resume();
+
+        // Generate an ID for the file, saving it with that filename
+        const path = config.get('Upload.path');
+        const newName = await identifier.generate();
+        const finalPath = path + '/' + newName;
+
+        // Set the file attributes
+        file = {
+            originalName: name,
+            name: newName,
+            path: finalPath,
+            encoding: encoding,
+            mime: mime
+        };
+
+        // Ensure the output directory exists
+        await disk.mkdir(path);
+
+        // Handle errors
+        stream.on('error', err => done(err, {path: finalPath}));
+        stream.on('limit', () => done('LIMIT_FILE_SIZE', {path: finalPath}));
+
+        file.size = await disk.write(finalPath, stream);
+
+        await done(null, {file: file});
+    });
+
+    busboy.on('error', err => done(err));
+    busboy.on('finished', () => done(null, {file: file}));
+
+    req.pipe(busboy);
+});
+
+module.exports = uploadMultipart;

app/util/upload/stats.js

@@ -0,0 +1,13 @@
+const ModelPath = '../../models/';
+const User = require(ModelPath + 'User.js');
+const Key = require(ModelPath + 'Key.js');
+
+const updateStats = req =>
+    Promise.all([
+        User.updateOne({username: req.username}, {$inc: {uploadCount: 1, uploadSize: req.file.size}}),
+        req.key
+            ? Key.updateOne({key: req.key}, {$inc: {uploadCount: 1, uploadSize: req.file.size}})
+            : Promise.resolve()
+    ]);
+
+module.exports = updateStats;

package-lock.json

@@ -152,11 +152,6 @@
         "buffer-equal": "^1.0.0"
       }
     },
-    "append-field": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/append-field/-/append-field-0.1.0.tgz",
-      "integrity": "sha1-bdxY+gg8e8VF08WZWygwzCNm1Eo="
-    },
     "archy": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz",
@@ -1249,6 +1244,7 @@
       "version": "1.6.0",
       "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.0.tgz",
       "integrity": "sha1-CqxmL9Ur54lk1VMvaUeE5wEQrPc=",
+      "dev": true,
       "requires": {
         "inherits": "^2.0.3",
         "readable-stream": "^2.2.2",
@@ -1258,12 +1254,14 @@
         "isarray": {
           "version": "1.0.0",
           "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-          "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
+          "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
+          "dev": true
         },
         "readable-stream": {
           "version": "2.3.3",
           "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.3.tgz",
           "integrity": "sha512-m+qzzcn7KUxEmd1gMbchF+Y2eIUbieUaxkWtptyHywrX0rE8QEYqPC07Vuy4Wm32/xE16NcdBctb8S0Xe/5IeQ==",
+          "dev": true,
           "requires": {
             "core-util-is": "~1.0.0",
             "inherits": "~2.0.3",
@@ -1278,6 +1276,7 @@
           "version": "1.0.3",
           "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz",
           "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==",
+          "dev": true,
           "requires": {
             "safe-buffer": "~5.1.0"
           }
@@ -5864,21 +5863,6 @@
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
       "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
     },
-    "multer": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/multer/-/multer-1.3.1.tgz",
-      "integrity": "sha512-JHdEoxkA/5NgZRo91RNn4UT+HdcJV9XUo01DTkKC7vo1erNIngtuaw9Y0WI8RdTlyi+wMIbunflhghzVLuGJyw==",
-      "requires": {
-        "append-field": "^0.1.0",
-        "busboy": "^0.2.11",
-        "concat-stream": "^1.5.2",
-        "mkdirp": "^0.5.1",
-        "object-assign": "^3.0.0",
-        "on-finished": "^2.3.0",
-        "type-is": "^1.6.4",
-        "xtend": "^4.0.0"
-      }
-    },
     "multipipe": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/multipipe/-/multipipe-0.1.2.tgz",
@@ -11910,7 +11894,8 @@
     "typedarray": {
       "version": "0.0.6",
       "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
-      "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c="
+      "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=",
+      "dev": true
     },
     "uglify-js": {
       "version": "3.4.5",

package.json

@@ -7,6 +7,7 @@
     "angular-ui-router": "^1.0.19",
     "async": "^2.6.1",
     "body-parser": "^1.18.3",
+    "busboy": "^0.2.14",
     "config": "^1.31.0",
     "connect-mongo": "^2.0.1",
     "express": "^4.16.3",
@@ -17,12 +18,12 @@
     "method-override": "latest",
     "mongoose": "^5.2.5",
     "morgan": "^1.9.0",
-    "multer": "^1.3.1",
     "ng-file-upload": "^12.2.13",
     "ngclipboard": "^2.0.0",
     "passport": "^0.4.0",
     "passport-local": "^1.0.0",
     "passport-local-mongoose": "^5.0.1",
+    "type-is": "^1.6.16",
     "vinyl-source-stream": "^2.0.0"
   },
   "description": "A simple file sharing website.",