Port Backend to SQLite/Sequelize (removes MongoDB), Support Electron (#14)

* The MongoDB/Mongoose data storage is removed in favor of Sequelize. This abstracts over SQLite or RDBMs like PostgreSQL and MSSQL. The default is SQLite, which significantly simplifies deployments in end-user environments.

* As Spacedeck now has no more mandatory server dependencies, we can wrap it in Electron and ship it as a desktop application.

* Removes docker-compose.yml

* First version of import UI

README.md

@@ -2,7 +2,7 @@
 
 This is the free and open source version of Spacedeck, a web based, real time, collaborative whiteboard application with rich media support. Spacedeck was developed in 6 major releases during Autumn 2011 until the end of 2016 and was originally a commercial SaaS. The developers were Lukas F. Hartmann (mntmn) and Martin Güther (magegu). All icons and large parts of the CSS were designed by Thomas Helbig (dergraph).
 
-As we plan to retire the subscription based service at spacedeck.com in late 2017, we decided to open-source Spacedeck to allow educational and other organizations who currently rely on Spacedeck to migrate to a self-hosted version.
+As we plan to retire the subscription based service at spacedeck.com in May 2018, we decided to open-source Spacedeck to allow educational and other organizations who currently rely on Spacedeck to migrate to a self-hosted or local version.
 
 Data migration features will be added soon.
 
@@ -23,20 +23,21 @@ We appreciate filed issues, pull requests and general discussion.
 
 Spacedeck uses the following major building blocks:
 
-- Vue.js: Frontend UI Framework
-- Node.js 7.x: Web Server / API
-- MongoDB 3.4: Data store *(important: newer versions than 3.4 don't work yet!)*
-- Redis 3.x: Data store for realtime channels, (*optional*)
+- Node.js 9.x: Web Server / API
+- Vue.js: Frontend UI Framework (included)
+- SQLite (included)
 
-It also has some binary dependencies for media conversion and PDF export:
+It also has some optional binary dependencies for advanced media conversion:
 
-- imagemagick, graphicsmagick, libav(+codecs, ffmpeg replacement), audiowaveform (https://github.com/bbcrd/audiowaveform), phantomjs (http://phantomjs.org/)
+- ffmpeg and ffprobe (for video/audio conversion)
+- audiowaveform (for audio waveform rendering) (https://github.com/bbcrd/audiowaveform)
+- ghostscript (gs, for PDF import)
 
 By default, media files are uploaded to the ```storage``` folder.
 
-Optionally, you can use Amazon S3 for file storage. In that case you need an Amazon AWS account and have the ```AWS_ACCESS_KEY_ID``` and ```AWS_SECRET_ACCESS_KEY``` environment variables defined. For sending emails in production, Amazon SES is required.
+To use Spacedeck, you only need Node.JS 9.x.
 
-To run Spacedeck, you need Node.JS 7.x and a running MongoDB 3.4 instance. Then, to install all node dependencies, run
+Then, to install all node dependencies, run
 
     npm install
 
@@ -48,18 +49,25 @@ To rebuild the frontend CSS styles (you need to do this at least once):
 
 See [config/default.json](config/default.json)
 
-# Run
+# Run (web server)
 
-    export NODE_ENV=development
-    npm start
-    open http://localhost:9666
+    node spacedeck.js
+
+Then open http://localhost:9666 in a web browser.
+
+# Run (desktop app with integrated web server)
+
+    electron .
 
 # License
 
-Spacedeck Open is released under the GNU Affero General Public License Version 3 (GNU AGPLv3).
+The Spacedeck logo and brand assets are registered trademarks of Spacedeck GmbH. All rights reserved.
+
+Spacedeck Open source code is released under the GNU Affero General Public License Version 3 (GNU AGPLv3).
 
     Spacedeck Open - Web-based Collaborative Whiteboard For Rich Media
-    Copyright (C) 2011-2017 Lukas F. Hartmann, Martin Güther, Thomas Helbig
+    Copyright (C) 2011-2018 Lukas F. Hartmann, Martin Güther
+    Icons and original CSS design copyright by Thomas Helbig
     
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU Affero General Public License as

app.js

-"use strict";
-
-require('./models/schema');
-require("log-timestamp");
-
-const config = require('config');
-const redis = require('./helpers/redis');
-const websockets = require('./helpers/websockets');
-
-const http = require('http');
-const path = require('path');
-
-const _ = require('underscore');
-const favicon = require('serve-favicon');
-const logger = require('morgan');
-const cookieParser = require('cookie-parser');
-const bodyParser = require('body-parser');
-
-const mongoose = require('mongoose');
-
-const swig = require('swig');
-const i18n = require('i18n-2');
-const helmet = require('helmet');
-
-const express = require('express');
-const app = express();
-const serveStatic = require('serve-static');
-
-const isProduction = app.get('env') === 'production';
-
-console.log("Booting Spacedeck Open… (environment: " + app.get('env') + ")");
-
-app.use(logger(isProduction ? 'combined' : 'dev'));
-
-i18n.expressBind(app, {
-  locales: ["en", "de", "fr"],
-  defaultLocale: "en",
-  cookieName: "spacedeck_locale",
-  devMode: (app.get('env') == 'development')
-});
-
-swig.setDefaults({
-  varControls: ["[[", "]]"] // otherwise it's not compatible with vue.js
-});
-
-swig.setFilter('cdn', function(input, idx) {
-  return input;
-});
-
-app.engine('html', swig.renderFile);
-app.set('view engine', 'html');
-
-if (isProduction) {
-  app.set('views', path.join(__dirname, 'build', 'views'));
-  app.use(favicon(path.join(__dirname, 'build', 'assets', 'images', 'favicon.png')));
-  app.use(express.static(path.join(__dirname, 'build', 'assets')));
-} else {
-  app.set('views', path.join(__dirname, 'views'));
-  app.use(favicon(path.join(__dirname, 'public', 'images', 'favicon.png')));
-  app.use(express.static(path.join(__dirname, 'public')));
-}
-
-app.use(bodyParser.json({
-  limit: '50mb'
-}));
-
-app.use(bodyParser.urlencoded({
-  extended: false,
-  limit: '50mb'
-}));
-
-app.use(cookieParser());
-app.use(helmet.frameguard())
-app.use(helmet.xssFilter())
-app.use(helmet.hsts({
-  maxAge: 7776000000,
-  includeSubdomains: true
-}))
-app.disable('x-powered-by');
-app.use(helmet.noSniff())
-
-// CUSTOM MIDDLEWARES
-
-app.use(require("./middlewares/templates"));
-app.use(require("./middlewares/error_helpers"));
-app.use(require("./middlewares/setuser"));
-app.use(require("./middlewares/cors"));
-app.use(require("./middlewares/i18n"));
-app.use("/api", require("./middlewares/api_helpers"));
-app.use('/api/spaces/:id', require("./middlewares/space_helpers"));
-app.use('/api/spaces/:id/artifacts/:artifact_id', require("./middlewares/artifact_helpers"));
-app.use('/api/teams', require("./middlewares/team_helpers"));
-
-// REAL ROUTES
-
-app.use('/api/users', require('./routes/api/users'));
-app.use('/api/memberships', require('./routes/api/memberships'));
-
-const spaceRouter = require('./routes/api/spaces');
-app.use('/api/spaces', spaceRouter);
-
-spaceRouter.use('/:id/artifacts', require('./routes/api/space_artifacts'));
-spaceRouter.use('/:id/memberships', require('./routes/api/space_memberships'));
-spaceRouter.use('/:id/messages', require('./routes/api/space_messages'));
-spaceRouter.use('/:id/digest', require('./routes/api/space_digest'));
-spaceRouter.use('/:id', require('./routes/api/space_exports'));
-
-app.use('/api/sessions', require('./routes/api/sessions'));
-app.use('/api/teams', require('./routes/api/teams'));
-app.use('/api/webgrabber', require('./routes/api/webgrabber'));
-app.use('/', require('./routes/root'));
-
-if (config.get('storage_local_path')) {
-  app.use('/storage', serveStatic(config.get('storage_local_path')+"/"+config.get('storage_bucket'), {
-    maxAge: 24*3600
-  }));
-}
-
-// catch 404 and forward to error handler
-app.use(require('./middlewares/404'));
-if (app.get('env') == 'development') {
-  app.set('view cache', false);
-  swig.setDefaults({cache: false});
-} else {
-  app.use(require('./middlewares/500'));
+const spacedeck = require('./spacedeck')
+
+const electron = require('electron')
+const electronApp = electron.app
+const BrowserWindow = electron.BrowserWindow
+let mainWindow
+
+function createWindow () {
+  mainWindow = new BrowserWindow({width: 1200, height: 700})
+  mainWindow.loadURL("http://localhost:9666")
+  mainWindow.on('closed', function () {
+    mainWindow = null
+  })
 }
 
-module.exports = app;
+electronApp.on('ready', createWindow)
 
-// CONNECT TO DATABASE
-const mongoHost = process.env.MONGO_PORT_27017_TCP_ADDR || config.get('mongodb_host');
-mongoose.connect('mongodb://' + mongoHost + '/spacedeck');
-
-// START WEBSERVER
-const port = 9666;
-
-const server = http.Server(app).listen(port, () => {
-  
-  if ("send" in process) {
-    process.send('online');
+// Quit when all windows are closed.
+electronApp.on('window-all-closed', function () {
+  // On OS X it is common for applications and their menu bar
+  // to stay active until the user quits explicitly with Cmd + Q
+  if (process.platform !== 'darwin') {
+    electronApp.quit()
   }
+})
 
-}).on('listening', () => {
-  
-  const host = server.address().address;
-  const port = server.address().port;
-  console.log('Spacedeck Open listening at http://%s:%s', host, port);
-
-}).on('error', (error) => {
-
-  if (error.syscall !== 'listen') {
-    throw error;
-  }
-
-  const bind = typeof port === 'string' ? 'Pipe ' + port : 'Port ' + port;
-  switch (error.code) {
-    case 'EACCES':
-      console.error(bind + ' requires elevated privileges');
-      process.exit(1);
-      break;
-    case 'EADDRINUSE':
-      console.error(bind + ' is already in use');
-      process.exit(1);
-      break;
-    default:
-      throw error;
-  }
-});
-
-//WEBSOCKETS & WORKER
-websockets.startWebsockets(server);
-redis.connectRedis();
-
-process.on('message', (message) => {
-  console.log("Process message:", message);
-  if (message === 'shutdown') {
-    console.log("Exiting spacedeck.");
-    process.exit(0);
+electronApp.on('activate', function () {
+  // On OS X it's common to re-create a window in the app when the
+  // dock icon is clicked and there are no other windows open.
+  if (mainWindow === null) {
+    createWindow()
   }
-});
+})

docker-compose.yml

-version: '2'
-services:
-  sync:
-    image: redis
-  storage:
-    image: minio/minio
-    environment:
-      - MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE
-      - MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
-    ports:
-      - 9123:9000
-    command: server /export
-  db:
-    image: mongo
-  spacedeck-open:
-    environment:
-      - env=development
-      - MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE
-      - MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
-    build: .
-    volumes:
-      # - ./:/usr/src/app
-      - /usr/src/app/node_modules
-    command: npm start
-    ports:
-      - 9666:9666
-    depends_on:
-      - db
-      - sync
-      - storage
-    links:
-      - storage
-      - db
-      - sync

helpers/artifact_converter.js

@@ -4,52 +4,18 @@ const exec = require('child_process');
 const gm = require('gm');
 const async = require('async');
 const fs = require('fs');
-const Models = require('../models/schema');
+const Models = require('../models/db');
 const uploader = require('../helpers/uploader');
 const path = require('path');
 const os = require('os');
 
-const fileExtensionMap = {
-  ".amr" : "audio/AMR",
-  ".ogg" : "audio/ogg",
-  ".aac" : "audio/aac",
-  ".mp3" : "audio/mpeg",
-  ".mpg" : "video/mpeg",
-  ".3ga" : "audio/3ga",
-  ".mp4" : "video/mp4",
-  ".wav" : "audio/wav",
-  ".mov" : "video/quicktime",
-  ".doc" : "application/msword",
-  ".dot" : "application/msword",
-  ".docx" : "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-  ".dotx" : "application/vnd.openxmlformats-officedocument.wordprocessingml.template",
-  ".docm" : "application/vnd.ms-word.document.macroEnabled.12",
-  ".dotm" : "application/vnd.ms-word.template.macroEnabled.12",
-  ".xls" : "application/vnd.ms-excel",
-  ".xlt" : "application/vnd.ms-excel",
-  ".xla" : "application/vnd.ms-excel",
-  ".xlsx" : "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-  ".xltx" : "application/vnd.openxmlformats-officedocument.spreadsheetml.template",
-  ".xlsm" : "application/vnd.ms-excel.sheet.macroEnabled.12",
-  ".xltm" : "application/vnd.ms-excel.template.macroEnabled.12",
-  ".xlam" : "application/vnd.ms-excel.addin.macroEnabled.12",
-  ".xlsb" : "application/vnd.ms-excel.sheet.binary.macroEnabled.12",
-  ".ppt" : "application/vnd.ms-powerpoint",
-  ".pot" : "application/vnd.ms-powerpoint",
-  ".pps" : "application/vnd.ms-powerpoint",
-  ".ppa" : "application/vnd.ms-powerpoint",
-  ".pptx" : "application/vnd.openxmlformats-officedocument.presentationml.presentation",
-  ".potx" : "application/vnd.openxmlformats-officedocument.presentationml.template",
-  ".ppsx" : "application/vnd.openxmlformats-officedocument.presentationml.slideshow",
-  ".ppam" : "application/vnd.ms-powerpoint.addin.macroEnabled.12",
-  ".pptm" : "application/vnd.ms-powerpoint.presentation.macroEnabled.12",
-  ".potm" : "application/vnd.ms-powerpoint.template.macroEnabled.12",
-  ".ppsm" : "application/vnd.ms-powerpoint.slideshow.macroEnabled.12",
-  ".key" : "application/x-iwork-keynote-sffkey",
-  ".pages" : "application/x-iwork-pages-sffpages",
-  ".numbers" : "application/x-iwork-numbers-sffnumbers",
-  ".ttf" : "application/x-font-ttf"
-};
+const db = require('../models/db');
+const Sequelize = require('sequelize');
+const Op = Sequelize.Op;
+
+const mime = require('mime-types');
+const fileType = require('file-type');
+const readChunk = require('read-chunk');
 
 const convertableImageTypes = [
   "image/png",
@@ -69,7 +35,7 @@ const convertableVideoTypes = [
 
 const convertableAudioTypes = [
   "application/ogg",
-  "audio/AMR",
+  "audio/amr",
   "audio/3ga",
   "audio/wav",
   "audio/3gpp",
@@ -128,7 +94,7 @@ function createWaveform(fileName, localFilePath, callback){
 
 function convertVideo(fileName, filePath, codec, callback, progress_callback) {
   var ext = path.extname(fileName);
-  var presetMime = fileExtensionMap[ext];
+  var presetMime = mime.lookup(fileName);
 
   var newExt = codec == "mp4" ? "mp4" : "ogv";
   var convertedPath = filePath + "." + newExt;
@@ -186,7 +152,7 @@ function convertVideo(fileName, filePath, codec, callback, progress_callback) {
 
 function convertAudio(fileName, filePath, codec, callback) {
   var ext = path.extname(fileName);
-  var presetMime = fileExtensionMap[ext];
+  var presetMime = mime.lookup(fileName);
 
   var newExt = codec == "mp3" ? "mp3" : "ogg";
   var convertedPath = filePath + "." + newExt;
@@ -223,22 +189,14 @@ function createThumbnailForVideo(fileName, filePath, callback) {
 
 function getMime(fileName, filePath, callback) {
   var ext = path.extname(fileName);
-  var presetMime = fileExtensionMap[ext];
+  var presetMime = mime.lookup(fileName);
   
   if (presetMime) {
     callback(null, presetMime);
   } else {
-    exec.execFile("file", ["-b","--mime-type", filePath], {}, function(error, stdout, stderr) {
-      console.log("file stdout: ",stdout);
-      if (stderr === '' && error == null) {
-        //filter special chars from commandline
-        var mime = stdout.replace(/[^a-zA-Z0-9\/\-]/g,'');
-        callback(null, mime);
-      } else {
-        console.log("getMime file error: ", error);
-        callback(error, null);
-      }
-    });
+    const buffer = readChunk.sync(filePath, 0, 4100);
+    var mimeType = fileType(buffer);
+    callback(null, mimeType);
   }
 }
 
@@ -272,7 +230,7 @@ function resizeAndUpload(a, size, max, fileName, localFilePath, callback) {
 }
 
 
-var resizeAndUploadImage = function(a, mime, size, fileName, fileNameOrg, imageFilePath, originalFilePath, payloadCallback) {
+var resizeAndUploadImage = function(a, mimeType, size, fileName, fileNameOrg, imageFilePath, originalFilePath, payloadCallback) {
   async.parallel({
     small: function(callback){
       resizeAndUpload(a, size, 320, fileName, imageFilePath, callback);
@@ -285,13 +243,13 @@ var resizeAndUploadImage = function(a, mime, size, fileName, fileNameOrg, imageF
     },
     original: function(callback){
       var s3Key = "s"+ a.space_id.toString() + "/a" + a._id + "/" + fileNameOrg;
-      uploader.uploadFile(s3Key, mime, originalFilePath, function(err, url){
+      uploader.uploadFile(s3Key, mimeType, originalFilePath, function(err, url){
         callback(null, url);
       });
     }
   }, function(err, results) {
     a.state = "idle";
-    a.mime = mime;
+    a.mime = mimeType;
     var stats = fs.statSync(originalFilePath);
 
     a.payload_size = stats["size"];
@@ -301,46 +259,41 @@ var resizeAndUploadImage = function(a, mime, size, fileName, fileNameOrg, imageF
     a.payload_uri = results.original;
 
     var factor = 320/size.width;
-    var newBoardSpecs = a.board;
-    newBoardSpecs.w = Math.round(size.width*factor);
-    newBoardSpecs.h = Math.round(size.height*factor);
-    a.board = newBoardSpecs;
+    a.w = Math.round(size.width*factor);
+    a.h = Math.round(size.height*factor);
 
     a.updated_at = new Date();
-    a.save(function(err) {
-      if(err) payloadCallback(err, null);
-      else {
-        fs.unlink(originalFilePath, function (err) {
-          if (err){
-            console.error(err);
-            payloadCallback(err, null);
-          } else {
-            console.log('successfully deleted ' + originalFilePath);
-            payloadCallback(null, a);
-          }
-        });
-      }
+    a.save().then(function() {
+      fs.unlink(originalFilePath, function (err) {
+        if (err){
+          console.error(err);
+          payloadCallback(err, null);
+        } else {
+          console.log('successfully deleted ' + originalFilePath);
+          payloadCallback(null, a);
+        }
+      });
     });
   });
 };
 
 module.exports = {
   convert: function(a, fileName, localFilePath, payloadCallback, progress_callback) {
-    getMime(fileName, localFilePath, function(err, mime){
-      console.log("[convert] fn: "+fileName+" local: "+localFilePath+" mime:", mime);
+    getMime(fileName, localFilePath, function(err, mimeType){
+      console.log("[convert] fn: "+fileName+" local: "+localFilePath+" mimeType:", mimeType);
 
       if (!err) {
-        if (convertableImageTypes.indexOf(mime) > -1) {
+        if (convertableImageTypes.indexOf(mimeType) > -1) {
          
           gm(localFilePath).size(function (err, size) {
             console.log("[convert] gm:", err, size);
 
             if (!err) {
-              if(mime == "application/pdf") {
+              if(mimeType == "application/pdf") {
                 var firstImagePath =  localFilePath + ".jpeg";
                 exec.execFile("gs", ["-sDEVICE=jpeg","-dNOPAUSE", "-dJPEGQ=80", "-dBATCH", "-dFirstPage=1", "-dLastPage=1", "-sOutputFile=" + firstImagePath, "-r90", "-f", localFilePath], {}, function(error, stdout, stderr) {
                   if(error === null) {
-                    resizeAndUploadImage(a, mime, size, fileName + ".jpeg", fileName, firstImagePath, localFilePath, function(err, a) {
+                    resizeAndUploadImage(a, mimeType, size, fileName + ".jpeg", fileName, firstImagePath, localFilePath, function(err, a) {
                       fs.unlink(firstImagePath, function (err) {
                         payloadCallback(err, a);
                       });
@@ -350,7 +303,7 @@ module.exports = {
                   }
                 });
 
-              } else if(mime == "image/gif") {
+              } else if(mimeType == "image/gif") {
                 //gifs are buggy after convertion, so we should not convert them
 
                 var s3Key = "s"+ a.space_id.toString() + "/a" + a._id.toString() + "/" + fileName;
@@ -362,7 +315,7 @@ module.exports = {
                     var stats = fs.statSync(localFilePath);
 
                     a.state = "idle";
-                    a.mime = mime;
+                    a.mime = mimeType;
 
                     a.payload_size = stats["size"];
                     a.payload_thumbnail_web_uri = url;
@@ -371,36 +324,31 @@ module.exports = {
                     a.payload_uri = url;
 
                     var factor = 320/size.width;
-                    var newBoardSpecs = a.board;
-                    newBoardSpecs.w = Math.round(size.width*factor);
-                    newBoardSpecs.h = Math.round(size.height*factor);
-                    a.board = newBoardSpecs;
+                    a.w = Math.round(size.width*factor);
+                    a.h = Math.round(size.height*factor);
 
                     a.updated_at = new Date();
-                    a.save(function(err){
-                      if(err) payloadCallback(err, null);
-                      else {
-                        fs.unlink(localFilePath, function (err) {
-                          if (err){
-                            console.error(err);
-                            payloadCallback(err, null);
-                          } else {
-                            console.log('successfully deleted ' + localFilePath);
-                            payloadCallback(null, a);
-                          }
-                        });
-                      }
+                    a.save().then(function() {
+                      fs.unlink(localFilePath, function (err) {
+                        if (err){
+                          console.error(err);
+                          payloadCallback(err, null);
+                        } else {
+                          console.log('successfully deleted ' + localFilePath);
+                          payloadCallback(null, a);
+                        }
+                      });
                     });
                   }
                 });
 
               } else {
-                resizeAndUploadImage(a, mime, size, fileName, fileName, localFilePath, localFilePath, payloadCallback);
+                resizeAndUploadImage(a, mimeType, size, fileName, fileName, localFilePath, localFilePath, payloadCallback);
               }
             } else payloadCallback(err);
           });            
       
-        } else if (convertableVideoTypes.indexOf(mime) > -1) {
+        } else if (convertableVideoTypes.indexOf(mimeType) > -1) {
           async.parallel({
             thumbnail: function(callback) {
               createThumbnailForVideo(fileName, localFilePath, function(err, created){
@@ -416,7 +364,7 @@ module.exports = {
               });
             },
             ogg: function(callback) {
-              if (mime == "video/ogg") {
+              if (mimeType == "video/ogg") {
                 callback(null, "org");
               } else {
                 convertVideo(fileName, localFilePath, "ogg", function(err, file) {
@@ -432,7 +380,7 @@ module.exports = {
               }
             },
             mp4: function(callback) {
-              if (mime == "video/mp4") {
+              if (mimeType == "video/mp4") {
                 callback(null, "org");
               } else {
                 convertVideo(fileName, localFilePath, "mp4", function(err, file) {
@@ -448,7 +396,7 @@ module.exports = {
               }
             },
             original: function(callback){
-              uploader.uploadFile(fileName, mime, localFilePath, function(err, url){
+              uploader.uploadFile(fileName, mimeType, localFilePath, function(err, url){
                 callback(null, url);
               });
             }
@@ -458,7 +406,7 @@ module.exports = {
             if (err) payloadCallback(err, a);
             else {
               a.state = "idle";
-              a.mime = mime;
+              a.mime = mimeType;
               var stats = fs.statSync(localFilePath);
 
               a.payload_size = stats["size"];
@@ -467,7 +415,7 @@ module.exports = {
               a.payload_thumbnail_big_uri = results.thumbnail;
               a.payload_uri = results.original;
 
-              if (mime == "video/mp4") {
+              if (mimeType == "video/mp4") {
                 a.payload_alternatives = [
                   {
                     mime: "video/ogg",
@@ -483,6 +431,8 @@ module.exports = {
                 ];
               }
 
+              db.packArtifact(a);
+
               a.updated_at = new Date();
               a.save(function(err) {
                 if (err) payloadCallback(err, null);
@@ -501,7 +451,7 @@ module.exports = {
             }
           });
 
-        } else if (convertableAudioTypes.indexOf(mime) > -1) {
+        } else if (convertableAudioTypes.indexOf(mimeType) > -1) {
 
           async.parallel({
             ogg: function(callback) {
@@ -539,7 +489,7 @@ module.exports = {
             },
             original: function(callback) {
               var keyName = "s" + a.space_id.toString() + "/a" + a._id.toString() + "/" + fileName;
-              uploader.uploadFile(keyName, mime, localFilePath, function(err, url){
+              uploader.uploadFile(keyName, mimeType, localFilePath, function(err, url){
                 callback(null, url);
               });
             }
@@ -550,7 +500,7 @@ module.exports = {
             else {
 
               a.state = "idle";
-              a.mime = mime;
+              a.mime = mimeType;
               var stats = fs.statSync(localFilePath);
 
               a.payload_size = stats["size"];
@@ -564,43 +514,40 @@ module.exports = {
               ];
 
               a.updated_at = new Date();
-              a.save(function(err){
-                if(err) payloadCallback(err, null);
-                else {
-                  fs.unlink(localFilePath, function (err) {
-                    if (err){
-                      console.error(err);
-                      payloadCallback(err, null);
-                    } else {
-                      console.log('successfully deleted ' + localFilePath);
-                      payloadCallback(null, a);
-                    }
-                  });
-                }
+
+              db.packArtifact(a);
+              
+              a.save().then(function(){
+                fs.unlink(localFilePath, function (err) {
+                  if (err){
+                    console.error(err);
+                    payloadCallback(err, null);
+                  } else {
+                    console.log('successfully deleted ' + localFilePath);
+                    payloadCallback(null, a);
+                  }
+                });
               });
             }
           });
 
 
         } else {
-          console.log("mime not matched for conversion, storing file");
+          console.log("mimeType not matched for conversion, storing file");
           var keyName = "s" + a.space_id.toString() + "/a" + a._id.toString() + "/" + fileName;
-          uploader.uploadFile(keyName, mime, localFilePath, function(err, url) {
+          uploader.uploadFile(keyName, mimeType, localFilePath, function(err, url) {
             
             a.state = "idle";
-            a.mime = mime;
+            a.mime = mimeType;
             var stats = fs.statSync(localFilePath);
             a.payload_size = stats["size"];
             a.payload_uri = url;
             
             a.updated_at = new Date();
-            a.save(function(err) {
-              if(err) payloadCallback(err, null);
-              else {
-                fs.unlink(localFilePath, function (err) {
-                  payloadCallback(null, a);
-                });
-              }
+            a.save().then(function() {
+              fs.unlink(localFilePath, function (err) {
+                payloadCallback(null, a);
+              });
             });
           });
         }

helpers/importer.js

@@ -5,7 +5,12 @@ const config = require('config')
 const fs = require('fs')
 const path = require('path')
 
-require('../models/schema')
+const db = require('../models/db')
+const Sequelize = require('sequelize')
+const Op = Sequelize.Op
+const uuidv4 = require('uuid/v4')
+
+require('../models/db')
 
 module.exports = {
   importZIP: function(user, zipPath) {
@@ -54,8 +59,8 @@ module.exports = {
           let artifacts = JSON.parse(fs.readFileSync(importDir+'/'+space._id+'_artifacts.json'))
           console.log('[import] space',space._id,'artifacts:',artifacts.length)
 
-          let q = {_id: space._id}
-          space.creator = user._id
+          //let q = {where: {_id: space._id}}
+          space.creator_id = user._id
           delete space.__v
           
           // transplant homefolder
@@ -64,17 +69,35 @@ module.exports = {
             space.parent_space_id = user.home_folder_id
           }
 
-          Space.findOneAndUpdate(q, space, {upsert: true}, function(err,res) {
-            if (err) console.log("[import] space upsert err:",err)
-          })
+          // move nested attrs
+          console.log(space)
+          for (k in space.advanced) {
+            space[k] = space.advanced[k]
+          }
+
+          db.Space.create(space)
+            .error((err) => {
+              console.error("[import] space upsert err:",err)
+            })
           
           for (var j=0; j<artifacts.length; j++) {
             let a = artifacts[j]
             
             let q = {_id: a._id}
-            a.creator = user._id
+            a.user_id = user._id
             delete a.__v
             delete a.payload_thumbnail_big_uri
+            
+            // move nested attrs
+            for (k in a.style) {
+              a[k] = a.style[k]
+            }
+            for (k in a.meta) {
+              a[k] = a.meta[k]
+            }
+            for (k in a.board) {
+              a[k] = a.board[k]
+            }
 
             let prefix = "/storage/"+relativeImportDir+"/"+space._id+"_files/"
             if (a.thumbnail_uri && a.thumbnail_uri[0]!='/') a.thumbnail_uri = prefix + a.thumbnail_uri
@@ -92,8 +115,10 @@ module.exports = {
               }
             }
 
-            Artifact.findOneAndUpdate(q, a, {upsert: true}, function(err,res) {
-              if (err) console.log("[import] artifact upsert err:",err)
+            db.packArtifact(a)
+
+            db.Artifact.create(a).error(function(err) {
+              console.error("[import] artifact upsert err:",err)
             })
           }
         }

helpers/mailer.js

 'use strict';
 
 var swig = require('swig');
-var AWS = require('aws-sdk');
+//var AWS = require('aws-sdk');
 
 module.exports = {
   sendMail: (to_email, subject, body, options) => {
@@ -29,9 +29,9 @@ module.exports = {
       options: options
     });
 
-    if (process.env.NODE_ENV === 'development') {
+    //if (process.env.NODE_ENV === 'development') {
       console.log("Email: to " + to_email + " in production.\nreply_to: " + reply_to + "\nsubject: " + subject + "\nbody: \n" + htmlText + "\n\n plaintext:\n" + plaintext);
-    } else {
+    /*} else {
       AWS.config.update({region: 'eu-west-1'});
       var ses = new AWS.SES();
 
@@ -56,6 +56,6 @@ module.exports = {
         if (err) console.error("Error sending email:", err);
         else console.log("Email sent.");
       });
-    }
+    }*/
   }
 };

helpers/phantom.js

 'use strict';
 
-require('../models/schema');
-var config = require('config');
-var phantom = require('node-phantom-simple');
+const db = require('../models/db');
+const config = require('config');
+const phantom = require('node-phantom-simple');
+const os = require('os');
 
 module.exports = {
   // type = "pdf" or "png"
@@ -10,7 +11,7 @@ module.exports = {
     var spaceId = space._id;
     var space_url = config.get("endpoint")+"/api/spaces/"+spaceId+"/html";
 
-    var export_path = "/tmp/"+spaceId+"."+type;
+    var export_path = os.tmpdir()+"/"+spaceId+"."+type;
 
     var timeout = 5000;
     if (type=="pdf") timeout = 30000;
@@ -24,7 +25,7 @@ module.exports = {
 
     var on_exit = function(exit_code) {
       if (exit_code>0) {
-        console.log("phantom abnormal exit for url "+space_url);
+        console.error("phantom abnormal exit for url "+space_url);
         if (!on_success_called && on_error) {
           on_error();
         }
@@ -32,16 +33,16 @@ module.exports = {
     };
 
     phantom.create({ path: require('phantomjs-prebuilt').path }, function (err, browser) {
-      if(err){
-        console.log(err);
-      }else{
+      if (err) {
+        console.error(err);
+      } else {
         return browser.createPage(function (err, page) {
           console.log("page created, opening ",space_url);
 
           if (type=="pdf") {
             var psz = {
-              width: space.advanced.width+"px",
-              height: space.advanced.height+"px"
+              width: space.width+"px",
+              height: space.height+"px"
             };
             page.set('paperSize', psz);
           }

helpers/websockets.js

 'use strict';
-require('../models/schema');
+
+const db = require('../models/db');
+const Sequelize = require('sequelize');
+const Op = Sequelize.Op;
 
 const config = require('config');
 
 const WebSocketServer = require('ws').Server;
 
-const RedisConnection = require('ioredis');
+//const RedisConnection = require('ioredis');
 const async = require('async');
 const _ = require("underscore");
-const mongoose = require("mongoose");
 const crypto = require('crypto');
 
 const redisMock = require("./redis.js");
@@ -45,11 +47,11 @@ module.exports = {
             const editorAuth = msg.editor_auth;
             const spaceId = msg.space_id;
 
-            Space.findOne({"_id": spaceId}).populate('creator').exec((err, space) => {
+            db.Space.findOne({where: {"_id": spaceId}}).then(space => {
               if (space) {
                 const upgradeSocket = function() {
                   if (token) {
-                    User.findBySessionToken(token, function(err, user) {
+                    db.findUserBySessionToken(token, function(err, user) {
                       if (err) {
                         console.error(err, user);
                       } else {
@@ -268,10 +270,10 @@ module.exports = {
   },
 
   distributeUsers: function(spaceId) {
-    if(!spaceId)
+    if (!spaceId)
       return;
 
-    this.state.smembers("space_" + spaceId, function(err, list) {
+    /*this.state.smembers("space_" + spaceId, function(err, list) {
       async.map(list, function(item, callback) {
         this.state.get(item, function(err, userId) {
           console.log(item, "->", userId);
@@ -292,16 +294,14 @@ module.exports = {
           return {nickname: realNickname, email: null, avatar_thumbnail_uri: null };
         });
 
-        User.find({"_id" : { "$in" : validUserIds }}, { "nickname" : 1 ,  "email" : 1, "avatar_thumbnail_uri": 1 }, function(err, users) {
-          if (err)
-            console.error(err);
-          else {
+        db.User.findAll({where: {
+          "_id" : { "$in" : validUserIds }}, attributes: ["nickname","email","avatar_thumbnail_uri"]})
+          .then(users) {
             const allUsers = users.concat(anonymousUsers);
             const strUsers = JSON.stringify({users: allUsers, space_id: spaceId});
             this.state.publish("users", strUsers);
-          }
-        }.bind(this));
+          }.bind(this));
       }.bind(this));
-    }.bind(this));
+    }.bind(this));*/
   }
 };

middlewares/404.js

 'use strict';
 
-require('../models/schema');
+require('../models/db');
 var config = require('config');
 
 module.exports = (req, res, next) => {
@@ -16,4 +16,4 @@ module.exports = (req, res, next) => {
   } else {
     res.status(404).send("Not Found.");
   }
-}
+}
\ No newline at end of file

middlewares/api_helpers.js

 'use strict';
 
-require('../models/schema');
+require('../models/db');
 var config = require('config');
 const redis = require('../helpers/redis');
 
+// FIXME TODO object.toJSON()
+
 var saveAction = (actionKey, object) => {
   if (object.constructor.modelName == "Space")
     return;
@@ -13,14 +15,14 @@ var saveAction = (actionKey, object) => {
     space: object.space_id || object.space,
     user: object.user_id || object.user,
     editor_name: object.editor_name,
-    object: object.toJSON()
+    object: object
   };
 
-  let action = new Action(attr);
+  /*let action = new Action(attr);
   action.save(function(err) {
     if (err)
       console.error("saved create action err:", err);
-  });
+  });*/
 };
 
 module.exports = (req, res, next) => {
@@ -32,21 +34,21 @@ module.exports = (req, res, next) => {
 
   res['distributeCreate'] = function(model, object) {
     if (!object) return;
-    redis.sendMessage("create", model, object.toJSON(), req.channelId);
-    this.status(201).json(object.toJSON());
+    redis.sendMessage("create", model, object, req.channelId);
+    this.status(201).json(object);
     saveAction("create", object);
   };
 
   res['distributeUpdate'] = function(model, object) {
     if (!object) return;
-    redis.sendMessage("update", model, object.toJSON(), req.channelId);
-    this.status(200).json(object.toJSON());
+    redis.sendMessage("update", model, object, req.channelId);
+    this.status(200).json(object);
     saveAction("update", object);
   };
 
   res['distributeDelete'] = function(model, object) {
     if (!object) return;
-    redis.sendMessage("delete", model, object.toJSON(), req.channelId);
+    redis.sendMessage("delete", model, object, req.channelId);
     this.sendStatus(204);
     saveAction("delete", object);
   };

middlewares/artifact_helpers.js

 'use strict';
+const db = require('../models/db');
+const Sequelize = require('sequelize');
+const Op = Sequelize.Op;
 
-require('../models/schema');
 var config = require('config');
 
 module.exports = (req, res, next) => {
   var artifactId = req.params.artifact_id;
-  Artifact.findOne({
+  db.Artifact.findOne({where: {
     "_id": artifactId
-  }, (err, artifact) => {
-    if (err) {
-      res.status(400).json(err);
+  }}).then(artifact => {
+    if (artifact) {
+      req['artifact'] = artifact;
+      next();
     } else {
-      if (artifact) {
-        req['artifact'] = artifact;
-        next();
-      } else {
-        res.sendStatus(404);
-      }
+      res.sendStatus(404);
     }
   });
-};
+};
\ No newline at end of file

middlewares/cors.js

 'use strict';
 
-require('../models/schema');
+require('../models/db');
 const config = require('config');
 const url = require('url');
 
@@ -26,20 +26,20 @@ module.exports = (req, res, next) => {
     const parsedUrl = url.parse(origin, true, true);
 
     // FIXME
-    if (parsedUrl.hostname == "cdn.spacedeck.com") {      
+    if (parsedUrl.hostname == "cdn.spacedeck.com") {
       res.header('Cache-Control', "max-age");
       res.header('Expires', "30d");
       res.removeHeader("Pragma");
 
       respond(origin, req, res, next);
     } else {
-      Team.getTeamForHost(parsedUrl.hostname, (err, team, subdomain) => {
-        if (team) {
+      //Team.getTeamForHost(parsedUrl.hostname, (err, team, subdomain) => {
+        //if (team) {
           respond(origin, req, res, next);
-        } else {
+        //} else {
           next();
-        }
-      });
+        //}
+      //});
     }
 
   } else {

middlewares/i18n.js

 'use strict';
 
-require('../models/schema');
+require('../models/db');
 var config = require('config');
 
 module.exports = (req, res, next) => {
@@ -10,8 +10,8 @@ module.exports = (req, res, next) => {
     req.i18n.setLocaleFromCookie();
   }
 
-  if (req.user && req.user.preferences.language) {
-    req.i18n.setLocale(req.user.preferences.language);
+  if (req.user && req.user.prefs_language) {
+    req.i18n.setLocale(req.user.prefs_language);
   }
   next();
-}
+}
\ No newline at end of file

middlewares/session.js

+'use strict';
+
+const db = require('../models/db');
+var config = require('config');
+
+module.exports = (req, res, next) => {
+  const token = req.cookies["sdsession"];
+  
+  if (token && token != "null" && token != null) {
+    db.Session.findOne({where: {token: token}})
+      .then(session => {
+        if (!session) {
+          // session not found
+          next();
+        }
+        else db.User.findOne({where: {_id: session.user_id}})
+          .then(user => {
+            if (!user) {
+              res.clearCookie('sdsession');
+
+              if (req.accepts("text/html")) {
+                res.send("Please clear your cookies and try again.");
+              } else if (req.accepts('application/json')) {
+                res.status(403).json({
+                  "error": "token_not_found"
+                });
+              } else {
+                res.send("Please clear your cookies and try again.");
+              }
+              
+            } else {
+              req["token"] = token;
+              req["user"] = user;
+              next();
+            }
+          });
+      })
+      .error(err => {
+        console.error("Session resolve error",err);
+        next();
+      });
+  } else {
+    next();
+  }
+}
+

middlewares/setuser.js

-'use strict';
-
-require('../models/schema');
-var config = require('config');
-
-module.exports = (req, res, next) => {
-  const token = req.cookies["sdsession"];
-  
-  if (token && token != "null" && token !== null) {
-    User.findOne({
-      "sessions.token": token
-    }).populate('team').exec((err, user) => {
-      if (err) console.error("session.token lookup error:",err);
-      if (!user) {
-        res.clearCookie('sdsession');
-
-        if (req.accepts("text/html")) {
-          res.send("Please clear your cookies and try again.");
-        } else if (req.accepts('application/json')) {
-          res.status(403).json({
-            "error": "token_not_found"
-          });
-        } else {
-          res.send("Please clear your cookies and try again.");
-        }
-      } else {
-        req["token"] = token;
-        req["user"] = user;
-        next();
-      }
-    });
-  } else {
-    next();
-  }
-}

middlewares/space_helpers.js

 'use strict';
 
-require('../models/schema');
+const db = require('../models/db');
 var config = require('config');
 
 module.exports = (req, res, next) => {
@@ -19,50 +19,6 @@ module.exports = (req, res, next) => {
     }
   };
 
-  var rolePerUser = (originalSpace, user, cb) => {
-    originalSpace.path = [];
-
-    if (originalSpace._id.equals(req.user.home_folder_id) || (originalSpace.creator && originalSpace.creator._id.equals(req.user._id))) {
-      cb("admin");
-    } else {
-      var findMembershipsForSpace = function(space, allMemberships, prevRole) {
-        Membership.find({
-          "space": space._id
-        }, function(err, parentMemberships) {
-          var currentMemberships = parentMemberships.concat(allMemberships);
-
-          if (space.parent_space_id) {
-            Space.findOne({
-              "_id": space.parent_space_id
-            }, function(err, parentSpace) {
-              findMembershipsForSpace(parentSpace, currentMemberships, prevRole);
-            });
-          } else {
-            // reached the top
-
-            var role = prevRole;
-            space.memberships = currentMemberships;
-
-            if(role == "none"){
-              if(originalSpace.access_mode == "public") {
-                role = "viewer";
-              }
-            }
-
-            currentMemberships.forEach(function(m, i) {
-              if (m.user && m.user.equals(user._id)) {
-                role = m.role;
-              }
-            });
-
-            cb(role);
-          }
-        });
-      };
-      findMembershipsForSpace(originalSpace, [], "none");
-    }
-  };
-
   var finalizeAnonymousLogin = function(space, spaceAuth) {
     var role = "none";
 
@@ -77,7 +33,7 @@ module.exports = (req, res, next) => {
     }
 
     if (req.user) {
-      rolePerUser(space, req.user, function(newRole) {
+      db.getUserRoleInSpace(space, req.user, function(newRole) {
         if (newRole == "admin" && (role == "editor" || role == "viewer")) {
           finalizeReq(space, newRole);
         } else if (newRole == "editor" && (role == "viewer")) {
@@ -97,64 +53,66 @@ module.exports = (req, res, next) => {
     'email': 1
   };
 
-  Space.findOne({
+  db.Space.findOne({where: {
     "_id": spaceId
-  }).populate("creator", userMapping).exec(function(err, space) {
-    if (err) {
-      res.status(400).json(err);
-    } else {
-      if (space) {
+  }}).then(function(space) {
 
-        if (space.access_mode == "public") {
+    //.populate("creator", userMapping)
+    //if (err) {
+    //  res.status(400).json(err);
+    //} else {
 
-          if (space.password) {
-            if (req.spacePassword) {
-              if (req.spacePassword === space.password) {
-                finalizeAnonymousLogin(space, req["spaceAuth"]);
-              } else {
-                res.status(403).json({
-                  "error": "password_wrong"
-                });
-              }
+    if (space) {
+      if (space.access_mode == "public") {
+        if (space.password) {
+          if (req.spacePassword) {
+            if (req.spacePassword === space.password) {
+              finalizeAnonymousLogin(space, req["spaceAuth"]);
             } else {
-              res.status(401).json({
-                "error": "password_required"
+              res.status(403).json({
+                "error": "password_wrong"
               });
             }
           } else {
-            finalizeAnonymousLogin(space, req["spaceAuth"]);
+            res.status(401).json({
+              "error": "password_required"
+            });
           }
-
         } else {
-          // special permission for screenshot/pdf export from backend
-          if (req.query['api_token'] && req.query['api_token'] == config.get('phantom_api_secret')) {
-            finalizeReq(space, "viewer");
-            return;
-          }
+          finalizeAnonymousLogin(space, req["spaceAuth"]);
+        }
 
-          if (req.user) {
-            rolePerUser(space, req.user, function(role) {
-              if (role == "none") {
-                finalizeAnonymousLogin(space, req["spaceAuth"]);
-              } else {
-                finalizeReq(space, role);
-              }
-            });
-          } else {
-            if (req.spaceAuth && space.edit_hash) {
+      } else {
+        // space is private
+        
+        // special permission for screenshot/pdf export from backend
+        if (req.query['api_token'] && req.query['api_token'] == config.get('phantom_api_secret')) {
+          finalizeReq(space, "viewer");
+          return;
+        }
+
+        if (req.user) {
+          db.getUserRoleInSpace(space, req.user, function(role) {
+            if (role == "none") {
               finalizeAnonymousLogin(space, req["spaceAuth"]);
             } else {
-              res.status(403).json({
-                "error": "auth_required"
-              });
+              finalizeReq(space, role);
             }
+          });
+        } else {
+          if (req.spaceAuth && space.edit_hash) {
+            finalizeAnonymousLogin(space, req["spaceAuth"]);
+          } else {
+            res.status(403).json({
+              "error": "auth_required"
+            });
           }
         }
-      } else {
-        res.status(404).json({
-          "error": "space_not_found"
-        });
       }
+    } else {
+      res.status(404).json({
+        "error": "space_not_found"
+      });
     }
   });
 }

middlewares/subdomain.js

-'use strict';
-
-require('../models/schema');
-var config = require('config');
-
-module.exports = (req, res, next) => {
-  let host = req.headers.host;
-  Team.getTeamForHost(host, (err, team, subdomain) => {
-    if (subdomain) {
-      if (!err && team) {
-        req.subdomainTeam = team;
-        req.subdomain = subdomain;
-        next()
-      } else {
-        if (req.accepts('text/html')) {
-          res.status(404).render('not_found', {
-            title: 'Page Not Found.'
-          });
-        } else if (req.accepts('application/json')) {
-          res.status(404).json({
-            "error": "not_found"
-          });
-        } else {
-          res.status(404).render('not_found', {
-            title: 'Page Not Found.'
-          });
-        }
-      }
-    } else {
-      next();
-    }
-  });
-}

middlewares/team_helpers.js

-'use strict';
-
-require('../models/schema');
-var config = require('config');
-
-module.exports = (req, res, next) => {
-  if (req.user) {
-    var isAdmin = req.user.team.admins.indexOf(req.user._id) >= 0;
-    var correctMethod = req.method == "GET" || (req.method == "DELETE" || req.method == "PUT" || req.method == "POST");
-
-    if (correctMethod && isAdmin) {
-      next();
-    } else {
-      res.status(403, {
-        "error": "not authorized"
-      });
-    }
-  } else {
-    res.status(403, {
-      "error": "not logged in"
-    });
-  }
-}
\ No newline at end of file

middlewares/templates.js

-'use strict';
-
-require('../models/schema');
-var config = require('config');
-var _ = require('underscore');
-
-module.exports = (req, res, next) => {
-  res.oldRender = res.render;
-  res.render = function(template, params) {
-
-    var team = req.subdomainTeam;
-    if (team) {
-      team = _.pick(team.toObject(), ['_id', 'name', 'subdomain', 'avatar_original_uri']);
-    } else {
-      team = null;
-    }
-
-    const addParams = {
-      locale: req.i18n.locale,
-      config: config,
-      subdomain_team: team,
-      user: req.user,
-      csrf_token: "",
-      socket_auth: req.token
-    };
-
-    const all = _.extend(params, addParams);
-    res.oldRender(template, all);
-  };
-  next();
-}
\ No newline at end of file

models/action.js

 'use strict';
 
+// FIXME port this last model
+
 var mongoose = require('mongoose');
 var Schema = mongoose.Schema;