Update migrations code

config/default.yaml

@@ -10,6 +10,8 @@ database:
   hostname: 'localhost'
   port: 5432
   suffix: '_dev'
+  username: peertube
+  password: peertube
 
 # From the project root directory
 storage:

server.js

@@ -20,6 +20,7 @@ const constants = require('./server/initializers/constants')
 const logger = require('./server/helpers/logger')
 // Initialize database and models
 const db = require('./server/initializers/database')
+db.init()
 
 // ----------- Checker -----------
 const checker = require('./server/initializers/checker')

server/initializers/checker.js

@@ -27,7 +27,7 @@ function checkConfig () {
 function checkMissedConfig () {
   const required = [ 'listen.port',
     'webserver.https', 'webserver.hostname', 'webserver.port',
-    'database.hostname', 'database.port', 'database.suffix',
+    'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password',
     'storage.certs', 'storage.videos', 'storage.logs', 'storage.thumbnails', 'storage.previews'
   ]
   const miss = []

server/initializers/constants.js

@@ -37,7 +37,9 @@ const CONFIG = {
   DATABASE: {
     DBNAME: 'peertube' + config.get('database.suffix'),
     HOSTNAME: config.get('database.hostname'),
-    PORT: config.get('database.port')
+    PORT: config.get('database.port'),
+    USERNAME: config.get('database.username'),
+    PASSWORD: config.get('database.password')
   },
   STORAGE: {
     CERT_DIR: path.join(__dirname, '..', '..', config.get('storage.certs')),
@@ -87,41 +89,7 @@ const FRIEND_SCORE = {
 
 // ---------------------------------------------------------------------------
 
-const MIGRATION_SCRIPTS = [
-  {
-    script: '0005-create-application',
-    version: 5
-  },
-  {
-    script: '0010-users-password',
-    version: 10
-  },
-  {
-    script: '0015-admin-role',
-    version: 15
-  },
-  {
-    script: '0020-requests-endpoint',
-    version: 20
-  },
-  {
-    script: '0025-video-filenames',
-    version: 25
-  },
-  {
-    script: '0030-video-magnet',
-    version: 30
-  },
-  {
-    script: '0035-url-to-host',
-    version: 35
-  },
-  {
-    script: '0040-video-remote-id',
-    version: 40
-  }
-]
-const LAST_SQL_SCHEMA_VERSION = (maxBy(MIGRATION_SCRIPTS, 'version'))['version']
+const LAST_MIGRATION_VERSION = 0
 
 // ---------------------------------------------------------------------------
 
@@ -197,8 +165,7 @@ module.exports = {
   CONFIG,
   CONSTRAINTS_FIELDS,
   FRIEND_SCORE,
-  LAST_SQL_SCHEMA_VERSION,
-  MIGRATION_SCRIPTS,
+  LAST_MIGRATION_VERSION,
   OAUTH_LIFETIME,
   PAGINATION_COUNT_DEFAULT,
   PODS_SCORE,

server/initializers/database.js

@@ -10,7 +10,11 @@ const utils = require('../helpers/utils')
 
 const database = {}
 
-const sequelize = new Sequelize(constants.CONFIG.DATABASE.DBNAME, 'peertube', 'peertube', {
+const dbname = constants.CONFIG.DATABASE.DBNAME
+const username = constants.CONFIG.DATABASE.USERNAME
+const password = constants.CONFIG.DATABASE.PASSWORD
+
+const sequelize = new Sequelize(dbname, username, password, {
   dialect: 'postgres',
   host: constants.CONFIG.DATABASE.HOSTNAME,
   port: constants.CONFIG.DATABASE.PORT,
@@ -26,33 +30,48 @@ const sequelize = new Sequelize(constants.CONFIG.DATABASE.DBNAME, 'peertube', 'p
   }
 })
 
-const modelDirectory = path.join(__dirname, '..', 'models')
-fs.readdir(modelDirectory, function (err, files) {
-  if (err) throw err
-
-  files.filter(function (file) {
-    if (file === 'utils.js') return false
-
-    return true
-  })
-  .forEach(function (file) {
-    const model = sequelize.import(path.join(modelDirectory, file))
-
-    database[model.name] = model
-  })
-
-  Object.keys(database).forEach(function (modelName) {
-    if ('associate' in database[modelName]) {
-      database[modelName].associate(database)
-    }
-  })
-
-  logger.info('Database is ready.')
-})
-
 database.sequelize = sequelize
 database.Sequelize = Sequelize
+database.init = init
 
 // ---------------------------------------------------------------------------
 
 module.exports = database
+
+// ---------------------------------------------------------------------------
+
+function init (silent, callback) {
+  if (!callback) {
+    callback = silent
+    silent = false
+  }
+
+  if (!callback) callback = function () {}
+
+  const modelDirectory = path.join(__dirname, '..', 'models')
+  fs.readdir(modelDirectory, function (err, files) {
+    if (err) throw err
+
+    files.filter(function (file) {
+      // For all models but not utils.js
+      if (file === 'utils.js') return false
+
+      return true
+    })
+    .forEach(function (file) {
+      const model = sequelize.import(path.join(modelDirectory, file))
+
+      database[model.name] = model
+    })
+
+    Object.keys(database).forEach(function (modelName) {
+      if ('associate' in database[modelName]) {
+        database[modelName].associate(database)
+      }
+    })
+
+    if (!silent) logger.info('Database is ready.')
+
+    return callback(null)
+  })
+}

server/initializers/installer.js

@@ -121,9 +121,8 @@ function createOAuthAdminIfNotExist (callback) {
       logger.info('Username: ' + username)
       logger.info('User password: ' + password)
 
-      logger.info('Creating Application collection.')
-      const application = db.Application.build({ sqlSchemaVersion: constants.LAST_SQL_SCHEMA_VERSION })
-      application.save().asCallback(callback)
+      logger.info('Creating Application table.')
+      db.Application.create({ migrationVersion: constants.LAST_MIGRATION_VERSION }).asCallback(callback)
     })
   })
 }

server/initializers/migrations/0005-create-application.js

@@ -1,17 +0,0 @@
-/*
-  Create the application collection in MongoDB.
-  Used to store the actual MongoDB scheme version.
-*/
-
-const mongoose = require('mongoose')
-
-const Application = mongoose.model('Application')
-
-exports.up = function (callback) {
-  const application = new Application()
-  application.save(callback)
-}
-
-exports.down = function (callback) {
-  throw new Error('Not implemented.')
-}

server/initializers/migrations/0005-example.js

@@ -0,0 +1,14 @@
+/*
+  This is just an example.
+*/
+
+const db = require('../database')
+
+// options contains the transaction
+exports.up = function (options, callback) {
+  // db.Application.create({ migrationVersion: 42 }, { transaction: options.transaction }).asCallback(callback)
+}
+
+exports.down = function (options, callback) {
+  throw new Error('Not implemented.')
+}

server/initializers/migrations/0010-users-password.js

@@ -1,22 +0,0 @@
-/*
-  Convert plain user password to encrypted user password.
-*/
-
-const eachSeries = require('async/eachSeries')
-const mongoose = require('mongoose')
-
-const User = mongoose.model('User')
-
-exports.up = function (callback) {
-  User.list(function (err, users) {
-    if (err) return callback(err)
-
-    eachSeries(users, function (user, callbackEach) {
-      user.save(callbackEach)
-    }, callback)
-  })
-}
-
-exports.down = function (callback) {
-  throw new Error('Not implemented.')
-}

server/initializers/migrations/0015-admin-role.js

@@ -1,16 +0,0 @@
-/*
-  Set the admin role to the root user.
-*/
-
-const constants = require('../constants')
-const mongoose = require('mongoose')
-
-const User = mongoose.model('User')
-
-exports.up = function (callback) {
-  User.update({ username: 'root' }, { role: constants.USER_ROLES.ADMIN }, callback)
-}
-
-exports.down = function (callback) {
-  throw new Error('Not implemented.')
-}

server/initializers/migrations/0020-requests-endpoint.js

@@ -1,15 +0,0 @@
-/*
-  Set the endpoint videos for requests.
-*/
-
-const mongoose = require('mongoose')
-
-const Request = mongoose.model('Request')
-
-exports.up = function (callback) {
-  Request.update({ }, { endpoint: 'videos' }, callback)
-}
-
-exports.down = function (callback) {
-  throw new Error('Not implemented.')
-}

server/initializers/migrations/0025-video-filenames.js

@@ -1,57 +0,0 @@
-/*
-  Rename thumbnails and video filenames to _id.extension
-*/
-
-const each = require('async/each')
-const fs = require('fs')
-const path = require('path')
-const mongoose = require('mongoose')
-
-const constants = require('../constants')
-const logger = require('../../helpers/logger')
-
-const Video = mongoose.model('Video')
-
-exports.up = function (callback) {
-  // Use of lean because the new Video scheme does not have filename field
-  Video.find({ filename: { $ne: null } }).lean().exec(function (err, videos) {
-    if (err) throw err
-
-    each(videos, function (video, callbackEach) {
-      const torrentName = video.filename + '.torrent'
-      const thumbnailName = video.thumbnail
-      const thumbnailExtension = path.extname(thumbnailName)
-      const videoName = video.filename
-      const videoExtension = path.extname(videoName)
-
-      const newTorrentName = video._id + '.torrent'
-      const newThumbnailName = video._id + thumbnailExtension
-      const newVideoName = video._id + videoExtension
-
-      const torrentsDir = constants.CONFIG.STORAGE.TORRENTS_DIR
-      const thumbnailsDir = constants.CONFIG.STORAGE.THUMBNAILS_DIR
-      const videosDir = constants.CONFIG.STORAGE.VIDEOS_DIR
-
-      logger.info('Renaming %s to %s.', torrentsDir + torrentName, torrentsDir + newTorrentName)
-      fs.renameSync(torrentsDir + torrentName, torrentsDir + newTorrentName)
-
-      logger.info('Renaming %s to %s.', thumbnailsDir + thumbnailName, thumbnailsDir + newThumbnailName)
-      fs.renameSync(thumbnailsDir + thumbnailName, thumbnailsDir + newThumbnailName)
-
-      logger.info('Renaming %s to %s.', videosDir + videoName, videosDir + newVideoName)
-      fs.renameSync(videosDir + videoName, videosDir + newVideoName)
-
-      Video.load(video._id, function (err, videoObj) {
-        if (err) return callbackEach(err)
-
-        videoObj.extname = videoExtension
-        videoObj.remoteId = null
-        videoObj.save(callbackEach)
-      })
-    }, callback)
-  })
-}
-
-exports.down = function (callback) {
-  throw new Error('Not implemented.')
-}

server/initializers/migrations/0030-video-magnet.js

@@ -1,32 +0,0 @@
-/*
-  Change video magnet structures
-*/
-
-const each = require('async/each')
-const magnet = require('magnet-uri')
-const mongoose = require('mongoose')
-
-const Video = mongoose.model('Video')
-
-exports.up = function (callback) {
-  // Use of lean because the new Video scheme does not have magnetUri field
-  Video.find({ }).lean().exec(function (err, videos) {
-    if (err) throw err
-
-    each(videos, function (video, callbackEach) {
-      const parsed = magnet.decode(video.magnetUri)
-      const infoHash = parsed.infoHash
-
-      Video.load(video._id, function (err, videoObj) {
-        if (err) return callbackEach(err)
-
-        videoObj.magnet.infoHash = infoHash
-        videoObj.save(callbackEach)
-      })
-    }, callback)
-  })
-}
-
-exports.down = function (callback) {
-  throw new Error('Not implemented.')
-}

server/initializers/migrations/0035-url-to-host.js

@@ -1,30 +0,0 @@
-/*
-  Change video magnet structures
-*/
-
-const each = require('async/each')
-const mongoose = require('mongoose')
-
-const Video = mongoose.model('Video')
-
-exports.up = function (callback) {
-  // Use of lean because the new Video scheme does not have podUrl field
-  Video.find({ }).lean().exec(function (err, videos) {
-    if (err) throw err
-
-    each(videos, function (video, callbackEach) {
-      Video.load(video._id, function (err, videoObj) {
-        if (err) return callbackEach(err)
-
-        const host = video.podUrl.split('://')[1]
-
-        videoObj.podHost = host
-        videoObj.save(callbackEach)
-      })
-    }, callback)
-  })
-}
-
-exports.down = function (callback) {
-  throw new Error('Not implemented.')
-}

server/initializers/migrations/0040-video-remote-id.js

@@ -1,59 +0,0 @@
-/*
-  Use remote id as identifier
-*/
-
-const map = require('lodash/map')
-const mongoose = require('mongoose')
-const readline = require('readline')
-
-const rl = readline.createInterface({
-  input: process.stdin,
-  output: process.stdout
-})
-
-const logger = require('../../helpers/logger')
-const friends = require('../../lib/friends')
-
-const Pod = mongoose.model('Pod')
-const Video = mongoose.model('Video')
-
-exports.up = function (callback) {
-  Pod.find({}).lean().exec(function (err, pods) {
-    if (err) return callback(err)
-
-    // We need to quit friends first
-    if (pods.length === 0) {
-      return setVideosRemoteId(callback)
-    }
-
-    const timeout = setTimeout(function () {
-      throw new Error('You need to enter a value!')
-    }, 10000)
-
-    rl.question('I am sorry but I need to quit friends for upgrading. Do you want to continue? (yes/*)', function (answer) {
-      if (answer !== 'yes') throw new Error('I cannot continue.')
-
-      clearTimeout(timeout)
-      rl.close()
-
-      const urls = map(pods, 'url')
-      logger.info('Saying goodbye to: ' + urls.join(', '))
-
-      setVideosRemoteId(function () {
-        friends.quitFriends(callback)
-      })
-    })
-  })
-}
-
-exports.down = function (callback) {
-  throw new Error('Not implemented.')
-}
-
-function setVideosRemoteId (callback) {
-  Video.update({ filename: { $ne: null } }, { remoteId: null }, function (err) {
-    if (err) throw err
-
-    Video.update({ filename: null }, { remoteId: mongoose.Types.ObjectId() }, callback)
-  })
-}

server/initializers/migrator.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const eachSeries = require('async/eachSeries')
+const fs = require('fs')
 const path = require('path')
 
 const constants = require('./constants')
@@ -12,35 +13,24 @@ const migrator = {
 }
 
 function migrate (callback) {
-  db.Application.loadSqlSchemaVersion(function (err, actualVersion) {
+  db.Application.loadMigrationVersion(function (err, actualVersion) {
     if (err) return callback(err)
 
-    // If there are a new mongo schemas
-    if (!actualVersion || actualVersion < constants.LAST_SQL_SCHEMA_VERSION) {
+    // If there are a new migration scripts
+    if (actualVersion < constants.LAST_MIGRATION_VERSION) {
       logger.info('Begin migrations.')
 
-      eachSeries(constants.MONGO_MIGRATION_SCRIPTS, function (entity, callbackEach) {
-        const versionScript = entity.version
-
-        // Do not execute old migration scripts
-        if (versionScript <= actualVersion) return callbackEach(null)
-
-        // Load the migration module and run it
-        const migrationScriptName = entity.script
-        logger.info('Executing %s migration script.', migrationScriptName)
-
-        const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName))
-        migrationScript.up(function (err) {
-          if (err) return callbackEach(err)
-
-          // Update the new mongo version schema
-          db.Application.updateSqlSchemaVersion(versionScript, callbackEach)
-        })
-      }, function (err) {
+      getMigrationScripts(function (err, migrationScripts) {
         if (err) return callback(err)
 
-        logger.info('Migrations finished. New SQL version schema: %s', constants.LAST_SQL_SCHEMA_VERSION)
-        return callback(null)
+        eachSeries(migrationScripts, function (entity, callbackEach) {
+          executeMigration(actualVersion, entity, callbackEach)
+        }, function (err) {
+          if (err) return callback(err)
+
+          logger.info('Migrations finished. New migration version schema: %s', constants.LAST_MIGRATION_VERSION)
+          return callback(null)
+        })
       })
     } else {
       return callback(null)
@@ -52,3 +42,57 @@ function migrate (callback) {
 
 module.exports = migrator
 
+// ---------------------------------------------------------------------------
+
+function getMigrationScripts (callback) {
+  fs.readdir(path.join(__dirname, 'migrations'), function (err, files) {
+    if (err) return callback(err)
+
+    const filesToMigrate = []
+
+    files.forEach(function (file) {
+      // Filename is something like 'version-blabla.js'
+      const version = file.split('-')[0]
+      filesToMigrate.push({
+        version,
+        script: file
+      })
+    })
+
+    return callback(err, filesToMigrate)
+  })
+}
+
+function executeMigration (actualVersion, entity, callback) {
+  const versionScript = entity.version
+
+  // Do not execute old migration scripts
+  if (versionScript <= actualVersion) return callback(null)
+
+  // Load the migration module and run it
+  const migrationScriptName = entity.script
+  logger.info('Executing %s migration script.', migrationScriptName)
+
+  const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName))
+
+  db.sequelize.transaction().asCallback(function (err, t) {
+    if (err) return callback(err)
+
+    migrationScript.up({ transaction: t }, function (err) {
+      if (err) {
+        t.rollback()
+        return callback(err)
+      }
+
+      // Update the new migration version
+      db.Application.updateMigrationVersion(versionScript, t, function (err) {
+        if (err) {
+          t.rollback()
+          return callback(err)
+        }
+
+        t.commit()
+      })
+    })
+  })
+}

server/models/application.js

@@ -1,15 +1,15 @@
 module.exports = function (sequelize, DataTypes) {
   const Application = sequelize.define('Application',
     {
-      sqlSchemaVersion: {
+      migrationVersion: {
         type: DataTypes.INTEGER,
         defaultValue: 0
       }
     },
     {
       classMethods: {
-        loadSqlSchemaVersion,
-        updateSqlSchemaVersion
+        loadMigrationVersion,
+        updateMigrationVersion
       }
     }
   )
@@ -19,18 +19,28 @@ module.exports = function (sequelize, DataTypes) {
 
 // ---------------------------------------------------------------------------
 
-function loadSqlSchemaVersion (callback) {
+function loadMigrationVersion (callback) {
   const query = {
-    attributes: [ 'sqlSchemaVersion' ]
+    attributes: [ 'migrationVersion' ]
   }
 
   return this.findOne(query).asCallback(function (err, data) {
-    const version = data ? data.sqlSchemaVersion : 0
+    const version = data ? data.migrationVersion : 0
 
     return callback(err, version)
   })
 }
 
-function updateSqlSchemaVersion (newVersion, callback) {
-  return this.update({ sqlSchemaVersion: newVersion }).asCallback(callback)
+function updateMigrationVersion (newVersion, transaction, callback) {
+  const options = {
+    where: {}
+  }
+
+  if (!callback) {
+    transaction = callback
+  } else {
+    options.transaction = transaction
+  }
+
+  return this.update({ migrationVersion: newVersion }, options).asCallback(callback)
 }

server/models/video.js

@@ -16,7 +16,7 @@ const modelUtils = require('./utils')
 // ---------------------------------------------------------------------------
 
 module.exports = function (sequelize, DataTypes) {
-// TODO: add indexes on searchable columns
+  // TODO: add indexes on searchable columns
   const Video = sequelize.define('Video',
     {
       id: {
@@ -50,6 +50,7 @@ module.exports = function (sequelize, DataTypes) {
 
         generateThumbnailFromBase64,
         getDurationFromFile,
+        list,
         listForApi,
         listByHostAndRemoteId,
         listOwnedAndPopulateAuthorAndTags,
@@ -310,6 +311,10 @@ function getDurationFromFile (videoPath, callback) {
   })
 }
 
+function list (callback) {
+  return this.find().asCallback()
+}
+
 function listForApi (start, count, sort, callback) {
   const query = {
     offset: start,

server/tests/utils/servers.js

@@ -103,7 +103,7 @@ function runServer (number, callback) {
       if (serverRunString[key] === false) dontContinue = true
     }
 
-    // If no, there is maybe one thing not already initialized (mongodb...)
+    // If no, there is maybe one thing not already initialized (client/user credentials generation...)
     if (dontContinue === true) return
 
     server.app.stdout.removeListener('data', onStdout)