diff --git a/gulp/gulp-tests.js b/gulp/gulp-tests.js index c3445874fa..d61ead1c98 100644 --- a/gulp/gulp-tests.js +++ b/gulp/gulp-tests.js @@ -3,9 +3,7 @@ import { exec } from 'child_process'; import gulp from 'gulp'; import os from 'os'; import nconf from 'nconf'; -import { - pipe, -} from './taskHelper'; +import { pipe } from './taskHelper'; import { getDevelopmentConnectionUrl, getDefaultConnectionOptions, @@ -21,15 +19,16 @@ const TEST_DB_URI = nconf.get('TEST_DB_URI'); const SANITY_TEST_COMMAND = 'npm run test:sanity'; const COMMON_TEST_COMMAND = 'npm run test:common'; const CONTENT_TEST_COMMAND = 'npm run test:content'; -const CONTENT_OPTIONS = { maxBuffer: 1024 * 500 }; +const LIMIT_MAX_BUFFER_OPTIONS = { maxBuffer: 1024 * 500 }; -/* Helper methods for reporting test summary */ +/* Helper method for reporting test summary */ const testResults = []; const testCount = (stdout, regexp) => { const match = stdout.match(regexp); return parseInt(match && (match[1] || 0), 10); }; +/* Helper methods to correctly run child test processes */ const testBin = (string, additionalEnvVariables = '') => { if (os.platform() === 'win32') { if (additionalEnvVariables !== '') { @@ -41,6 +40,15 @@ const testBin = (string, additionalEnvVariables = '') => { return `NODE_ENV=test ${additionalEnvVariables} ${string}`; }; +function runInChildProcess (command, options = {}, envVariables = '') { + return done => pipe(exec(testBin(command, envVariables), options, done)); +} + +function integrationTestCommand (testDir, coverageDir) { + return `istanbul cover --dir coverage/${coverageDir} --report lcovonly node_modules/mocha/bin/_mocha -- ${testDir} --recursive --require ./test/helpers/start-server`; +} + +/* Test task definitions */ gulp.task('test:nodemon', gulp.series(done => { process.env.PORT = TEST_SERVER_PORT; // eslint-disable-line no-process-env process.env.NODE_DB_URI = TEST_DB_URI; // eslint-disable-line no-process-env @@ -82,31 +90,9 @@ gulp.task('test:prepare', gulp.series( done => done(), )); -gulp.task('test:sanity', cb => { - const runner = exec( - testBin(SANITY_TEST_COMMAND), - err => { - if (err) { - process.exit(1); - } - cb(); - }, - ); - pipe(runner); -}); +gulp.task('test:sanity', runInChildProcess(SANITY_TEST_COMMAND)); -gulp.task('test:common', gulp.series('test:prepare:build', cb => { - const runner = exec( - testBin(COMMON_TEST_COMMAND), - err => { - if (err) { - process.exit(1); - } - cb(); - }, - ); - pipe(runner); -})); +gulp.task('test:common', gulp.series('test:prepare:build', runInChildProcess(COMMON_TEST_COMMAND))); gulp.task('test:common:clean', cb => { pipe(exec(testBin(COMMON_TEST_COMMAND), () => cb())); @@ -130,22 +116,11 @@ gulp.task('test:common:safe', gulp.series('test:prepare:build', cb => { pipe(runner); })); -gulp.task('test:content', gulp.series('test:prepare:build', cb => { - const runner = exec( - testBin(CONTENT_TEST_COMMAND), - CONTENT_OPTIONS, - err => { - if (err) { - process.exit(1); - } - cb(); - }, - ); - pipe(runner); -})); +gulp.task('test:content', gulp.series('test:prepare:build', + runInChildProcess(CONTENT_TEST_COMMAND, LIMIT_MAX_BUFFER_OPTIONS))); gulp.task('test:content:clean', cb => { - pipe(exec(testBin(CONTENT_TEST_COMMAND), CONTENT_OPTIONS, () => cb())); + pipe(exec(testBin(CONTENT_TEST_COMMAND), LIMIT_MAX_BUFFER_OPTIONS, () => cb())); }); gulp.task('test:content:watch', gulp.series('test:content:clean', () => gulp.watch(['common/script/content/**', 'test/**'], gulp.series('test:content:clean', done => done())))); @@ -153,7 +128,7 @@ gulp.task('test:content:watch', gulp.series('test:content:clean', () => gulp.wat gulp.task('test:content:safe', gulp.series('test:prepare:build', cb => { const runner = exec( testBin(CONTENT_TEST_COMMAND), - CONTENT_OPTIONS, + LIMIT_MAX_BUFFER_OPTIONS, (err, stdout) => { // eslint-disable-line handle-callback-err testResults.push({ suite: 'Content Specs\t', @@ -167,76 +142,39 @@ gulp.task('test:content:safe', gulp.series('test:prepare:build', cb => { pipe(runner); })); -gulp.task('test:api:unit:run', done => { - const runner = exec( - testBin('istanbul cover --dir coverage/api-unit node_modules/mocha/bin/_mocha -- test/api/unit --recursive --require ./test/helpers/start-server'), - err => { - if (err) { - process.exit(1); - } - done(); - }, - ); - - pipe(runner); -}); +gulp.task('test:api:unit:run', + runInChildProcess(integrationTestCommand('test/api/unit', 'coverage/api-unit'))); gulp.task('test:api:unit:watch', () => gulp.watch(['website/server/libs/*', 'test/api/unit/**/*', 'website/server/controllers/**/*'], gulp.series('test:api:unit:run', done => done()))); -gulp.task('test:api-v3:integration', gulp.series('test:prepare:mongo', done => { - const runner = exec( - testBin('istanbul cover --dir coverage/api-v3-integration --report lcovonly node_modules/mocha/bin/_mocha -- test/api/v3/integration --recursive --require ./test/helpers/start-server'), - { maxBuffer: 500 * 1024 }, - err => { - if (err) { - process.exit(1); - } - done(); - }, - ); - - pipe(runner); -})); +gulp.task('test:api-v3:integration', gulp.series('test:prepare:mongo', + runInChildProcess( + integrationTestCommand('test/api/v3/integration', 'coverage/api-v3-integration'), + LIMIT_MAX_BUFFER_OPTIONS, + ))); gulp.task('test:api-v3:integration:watch', () => gulp.watch([ 'website/server/controllers/api-v3/**/*', 'common/script/ops/*', 'website/server/libs/*.js', 'test/api/v3/integration/**/*', ], gulp.series('test:api-v3:integration', done => done()))); -gulp.task('test:api-v3:integration:separate-server', done => { - const runner = exec( - testBin('mocha test/api/v3/integration --recursive --require ./test/helpers/start-server', 'LOAD_SERVER=0'), - { maxBuffer: 500 * 1024 }, - err => done(err), - ); +gulp.task('test:api-v3:integration:separate-server', runInChildProcess( + 'mocha test/api/v3/integration --recursive --require ./test/helpers/start-server', + LIMIT_MAX_BUFFER_OPTIONS, + 'LOAD_SERVER=0', +)); - pipe(runner); -}); +gulp.task('test:api-v4:integration', gulp.series('test:prepare:mongo', + runInChildProcess( + integrationTestCommand('test/api/v4', 'api-v4-integration'), + LIMIT_MAX_BUFFER_OPTIONS, + ))); -gulp.task('test:api-v4:integration', gulp.series('test:prepare:mongo', done => { - const runner = exec( - testBin('istanbul cover --dir coverage/api-v4-integration --report lcovonly node_modules/mocha/bin/_mocha -- test/api/v4 --recursive --require ./test/helpers/start-server'), - { maxBuffer: 500 * 1024 }, - err => { - if (err) { - process.exit(1); - } - done(); - }, - ); - - pipe(runner); -})); - -gulp.task('test:api-v4:integration:separate-server', done => { - const runner = exec( - testBin('mocha test/api/v4 --recursive --require ./test/helpers/start-server', 'LOAD_SERVER=0'), - { maxBuffer: 500 * 1024 }, - err => done(err), - ); - - pipe(runner); -}); +gulp.task('test:api-v4:integration:separate-server', runInChildProcess( + 'mocha test/api/v4 --recursive --require ./test/helpers/start-server', + LIMIT_MAX_BUFFER_OPTIONS, + 'LOAD_SERVER=0', +)); gulp.task('test:api:unit', gulp.series( 'test:prepare:mongo', diff --git a/test/api/unit/libs/xmlMarshaller.test.js b/test/api/unit/libs/xmlMarshaller.test.js new file mode 100644 index 0000000000..73633a4c30 --- /dev/null +++ b/test/api/unit/libs/xmlMarshaller.test.js @@ -0,0 +1,44 @@ +import * as xmlMarshaller from '../../../../website/server/libs/xmlMarshaller'; + +describe('xml marshaller marshalls user data', () => { + const minimumUser = { + pinnedItems: [], + unpinnedItems: [], + inbox: {}, + }; + + function userDataWith (fields) { + return { ...minimumUser, ...fields }; + } + + it('maps the newMessages field to have id as a value in a list.', () => { + const userData = userDataWith({ + newMessages: { + '283171a5-422c-4991-bc78-95b1b5b51629': { + name: 'The Language Hackers', + value: true, + }, + '283171a6-422c-4991-bc78-95b1b5b51629': { + name: 'The Bug Hackers', + value: false, + }, + }, + }); + + const xml = xmlMarshaller.marshallUserData(userData); + + expect(xml).to.equal(` + + + 283171a5-422c-4991-bc78-95b1b5b51629 + The Language Hackers + true + + + 283171a6-422c-4991-bc78-95b1b5b51629 + The Bug Hackers + false + +`); + }); +}); diff --git a/website/server/controllers/top-level/dataexport.js b/website/server/controllers/top-level/dataexport.js index 2a17f4e5e4..fd9a10c8bf 100644 --- a/website/server/controllers/top-level/dataexport.js +++ b/website/server/controllers/top-level/dataexport.js @@ -1,14 +1,12 @@ import _ from 'lodash'; import moment from 'moment'; -import * as js2xml from 'js2xmlparser'; // import Pageres from 'pageres'; // import nconf from 'nconf'; // import got from 'got'; import md from 'habitica-markdown'; import csvStringify from '../../libs/csvStringify'; -import { - NotFound, -} from '../../libs/errors'; +import { marshallUserData } from '../../libs/xmlMarshaller'; +import { NotFound } from '../../libs/errors'; import * as Tasks from '../../models/task'; import * as inboxLib from '../../libs/inbox'; // import { model as User } from '../../models/user'; @@ -85,7 +83,7 @@ api.exportUserHistory = { // Convert user to json and attach tasks divided by type and inbox messages // at user.tasks[`${taskType}s`] (user.tasks.{dailys/habits/...}) -async function _getUserDataForExport (user, xmlMode = false) { +async function _getUserDataForExport (user) { const userData = user.toJSON(); userData.tasks = {}; @@ -108,29 +106,6 @@ async function _getUserDataForExport (user, xmlMode = false) { userData.tasks[`${taskType}s`] = tasksPerType; }); - if (xmlMode) { - // object maps cant be parsed - userData.inbox.messages = _(userData.inbox.messages) - .map(m => { - const flags = Object.keys(m.flags); - m.flags = flags; - - return m; - }) - .value(); - - // _id gets parsed as an bytearray => which gets casted to a chararray => "weird chars" - userData.unpinnedItems = userData.unpinnedItems.map(i => ({ - path: i.path, - type: i.type, - })); - - userData.pinnedItems = userData.pinnedItems.map(i => ({ - path: i.path, - type: i.type, - })); - } - return userData; } @@ -171,13 +146,8 @@ api.exportUserDataXml = { url: '/export/userdata.xml', middlewares: [authWithSession], async handler (req, res) { - const userData = await _getUserDataForExport(res.locals.user, true); - const xmlData = js2xml.parse('user', userData, { - cdataInvalidChars: true, - declaration: { - include: false, - }, - }); + const userData = await _getUserDataForExport(res.locals.user); + const xmlData = marshallUserData(userData); res.set({ 'Content-Type': 'text/xml', diff --git a/website/server/libs/xmlMarshaller.js b/website/server/libs/xmlMarshaller.js new file mode 100644 index 0000000000..5e7bb96095 --- /dev/null +++ b/website/server/libs/xmlMarshaller.js @@ -0,0 +1,32 @@ +import _ from 'lodash'; +import * as js2xml from 'js2xmlparser'; + +export function marshallUserData (userData) { + // object maps can't be marshalled to XML + userData.inbox.messages = _(userData.inbox.messages) + .map(m => { + m.flags = Object.keys(m.flags); + return m; + }) + .value(); + + userData.newMessages = _.map(userData.newMessages, (msg, id) => ({ id, ...msg })); + + // _id gets parsed as a bytearray => which gets cast to a chararray => "weird chars" + userData.unpinnedItems = userData.unpinnedItems.map(i => ({ + path: i.path, + type: i.type, + })); + + userData.pinnedItems = userData.pinnedItems.map(i => ({ + path: i.path, + type: i.type, + })); + + return js2xml.parse('user', userData, { + cdataInvalidChars: true, + declaration: { + include: false, + }, + }); +} diff --git a/website/server/models/user/schema.js b/website/server/models/user/schema.js index f671479dc9..abc6426a56 100644 --- a/website/server/models/user/schema.js +++ b/website/server/models/user/schema.js @@ -1,4 +1,4 @@ -import mongoose from 'mongoose'; +import { Schema } from 'mongoose'; import validator from 'validator'; import shared from '../../../common'; import { // eslint-disable-line import/no-cycle @@ -10,8 +10,6 @@ import { schema as TagSchema } from '../tag'; import { schema as UserNotificationSchema } from '../userNotification'; import { schema as WebhookSchema } from '../webhook'; -const { Schema } = mongoose; - const RESTRICTED_EMAIL_DOMAINS = Object.freeze(['habitica.com', 'habitrpg.com']); // User schema definition