Kai/habitica
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'benkelaar-12482-xml-export' into develop

Browse Source
This commit is contained in:
Matteo Pagliazzi
2020-09-25 11:21:35 +02:00
parent b056763f09 8219dfd9e6
commit 6c6e8f0001
5 changed files with 123 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gulp/gulp-tests.js
+41 -103
View File
@@ -3,9 +3,7 @@ import { exec } from 'child_process';
import gulp from 'gulp';
import os from 'os';
import nconf from 'nconf';
import {
pipe,
} from './taskHelper';
import { pipe } from './taskHelper';
import {
getDevelopmentConnectionUrl,
getDefaultConnectionOptions,
@@ -21,15 +19,16 @@ const TEST_DB_URI = nconf.get('TEST_DB_URI');
const SANITY_TEST_COMMAND = 'npm run test:sanity';
const COMMON_TEST_COMMAND = 'npm run test:common';
const CONTENT_TEST_COMMAND = 'npm run test:content';
const CONTENT_OPTIONS = { maxBuffer: 1024 * 500 };
const LIMIT_MAX_BUFFER_OPTIONS = { maxBuffer: 1024 * 500 };
/* Helper methods for reporting test summary */
/* Helper method for reporting test summary */
const testResults = [];
const testCount = (stdout, regexp) => {
const match = stdout.match(regexp);
return parseInt(match && (match[1] || 0), 10);
};
/* Helper methods to correctly run child test processes */
const testBin = (string, additionalEnvVariables = '') => {
if (os.platform() === 'win32') {
if (additionalEnvVariables !== '') {
@@ -41,6 +40,15 @@ const testBin = (string, additionalEnvVariables = '') => {
return `NODE_ENV=test ${additionalEnvVariables} ${string}`;
};
function runInChildProcess (command, options = {}, envVariables = '') {
return done => pipe(exec(testBin(command, envVariables), options, done));
}
function integrationTestCommand (testDir, coverageDir) {
return `istanbul cover --dir coverage/${coverageDir} --report lcovonly node_modules/mocha/bin/_mocha -- ${testDir} --recursive --require ./test/helpers/start-server`;
}
/* Test task definitions */
gulp.task('test:nodemon', gulp.series(done => {
process.env.PORT = TEST_SERVER_PORT; // eslint-disable-line no-process-env
process.env.NODE_DB_URI = TEST_DB_URI; // eslint-disable-line no-process-env
@@ -82,31 +90,9 @@ gulp.task('test:prepare', gulp.series(
done => done(),
));
gulp.task('test:sanity', cb => {
const runner = exec(
testBin(SANITY_TEST_COMMAND),
err => {
if (err) {
process.exit(1);
}
cb();
},
);
pipe(runner);
});
gulp.task('test:sanity', runInChildProcess(SANITY_TEST_COMMAND));
gulp.task('test:common', gulp.series('test:prepare:build', cb => {
const runner = exec(
testBin(COMMON_TEST_COMMAND),
err => {
if (err) {
process.exit(1);
}
cb();
},
);
pipe(runner);
}));
gulp.task('test:common', gulp.series('test:prepare:build', runInChildProcess(COMMON_TEST_COMMAND)));
gulp.task('test:common:clean', cb => {
pipe(exec(testBin(COMMON_TEST_COMMAND), () => cb()));
@@ -130,22 +116,11 @@ gulp.task('test:common:safe', gulp.series('test:prepare:build', cb => {
pipe(runner);
}));
gulp.task('test:content', gulp.series('test:prepare:build', cb => {
const runner = exec(
testBin(CONTENT_TEST_COMMAND),
CONTENT_OPTIONS,
err => {
if (err) {
process.exit(1);
}
cb();
},
);
pipe(runner);
}));
gulp.task('test:content', gulp.series('test:prepare:build',
runInChildProcess(CONTENT_TEST_COMMAND, LIMIT_MAX_BUFFER_OPTIONS)));
gulp.task('test:content:clean', cb => {
pipe(exec(testBin(CONTENT_TEST_COMMAND), CONTENT_OPTIONS, () => cb()));
pipe(exec(testBin(CONTENT_TEST_COMMAND), LIMIT_MAX_BUFFER_OPTIONS, () => cb()));
});
gulp.task('test:content:watch', gulp.series('test:content:clean', () => gulp.watch(['common/script/content/**', 'test/**'], gulp.series('test:content:clean', done => done()))));
@@ -153,7 +128,7 @@ gulp.task('test:content:watch', gulp.series('test:content:clean', () => gulp.wat
gulp.task('test:content:safe', gulp.series('test:prepare:build', cb => {
const runner = exec(
testBin(CONTENT_TEST_COMMAND),
CONTENT_OPTIONS,
LIMIT_MAX_BUFFER_OPTIONS,
(err, stdout) => { // eslint-disable-line handle-callback-err
testResults.push({
suite: 'Content Specs\t',
@@ -167,76 +142,39 @@ gulp.task('test:content:safe', gulp.series('test:prepare:build', cb => {
pipe(runner);
}));
gulp.task('test:api:unit:run', done => {
const runner = exec(
testBin('istanbul cover --dir coverage/api-unit node_modules/mocha/bin/_mocha -- test/api/unit --recursive --require ./test/helpers/start-server'),
err => {
if (err) {
process.exit(1);
}
done();
},
);
pipe(runner);
});
gulp.task('test:api:unit:run',
runInChildProcess(integrationTestCommand('test/api/unit', 'coverage/api-unit')));
gulp.task('test:api:unit:watch', () => gulp.watch(['website/server/libs/*', 'test/api/unit/**/*', 'website/server/controllers/**/*'], gulp.series('test:api:unit:run', done => done())));
gulp.task('test:api-v3:integration', gulp.series('test:prepare:mongo', done => {
const runner = exec(
testBin('istanbul cover --dir coverage/api-v3-integration --report lcovonly node_modules/mocha/bin/_mocha -- test/api/v3/integration --recursive --require ./test/helpers/start-server'),
{ maxBuffer: 500 * 1024 },
err => {
if (err) {
process.exit(1);
}
done();
},
);
pipe(runner);
}));
gulp.task('test:api-v3:integration', gulp.series('test:prepare:mongo',
runInChildProcess(
integrationTestCommand('test/api/v3/integration', 'coverage/api-v3-integration'),
LIMIT_MAX_BUFFER_OPTIONS,
)));
gulp.task('test:api-v3:integration:watch', () => gulp.watch([
'website/server/controllers/api-v3/**/*', 'common/script/ops/*', 'website/server/libs/*.js',
'test/api/v3/integration/**/*',
], gulp.series('test:api-v3:integration', done => done())));
gulp.task('test:api-v3:integration:separate-server', done => {
const runner = exec(
testBin('mocha test/api/v3/integration --recursive --require ./test/helpers/start-server', 'LOAD_SERVER=0'),
{ maxBuffer: 500 * 1024 },
err => done(err),
);
gulp.task('test:api-v3:integration:separate-server', runInChildProcess(
'mocha test/api/v3/integration --recursive --require ./test/helpers/start-server',
LIMIT_MAX_BUFFER_OPTIONS,
'LOAD_SERVER=0',
));
pipe(runner);
});
gulp.task('test:api-v4:integration', gulp.series('test:prepare:mongo',
runInChildProcess(
integrationTestCommand('test/api/v4', 'api-v4-integration'),
LIMIT_MAX_BUFFER_OPTIONS,
)));
gulp.task('test:api-v4:integration', gulp.series('test:prepare:mongo', done => {
const runner = exec(
testBin('istanbul cover --dir coverage/api-v4-integration --report lcovonly node_modules/mocha/bin/_mocha -- test/api/v4 --recursive --require ./test/helpers/start-server'),
{ maxBuffer: 500 * 1024 },
err => {
if (err) {
process.exit(1);
}
done();
},
);
pipe(runner);
}));
gulp.task('test:api-v4:integration:separate-server', done => {
const runner = exec(
testBin('mocha test/api/v4 --recursive --require ./test/helpers/start-server', 'LOAD_SERVER=0'),
{ maxBuffer: 500 * 1024 },
err => done(err),
);
pipe(runner);
});
gulp.task('test:api-v4:integration:separate-server', runInChildProcess(
'mocha test/api/v4 --recursive --require ./test/helpers/start-server',
LIMIT_MAX_BUFFER_OPTIONS,
'LOAD_SERVER=0',
));
gulp.task('test:api:unit', gulp.series(
'test:prepare:mongo',
test/api/unit/libs/xmlMarshaller.test.js
+44
View File
@@ -0,0 +1,44 @@
import * as xmlMarshaller from '../../../../website/server/libs/xmlMarshaller';
describe('xml marshaller marshalls user data', () => {
const minimumUser = {
pinnedItems: [],
unpinnedItems: [],
inbox: {},
};
function userDataWith (fields) {
return { ...minimumUser, ...fields };
}
it('maps the newMessages field to have id as a value in a list.', () => {
const userData = userDataWith({
newMessages: {
'283171a5-422c-4991-bc78-95b1b5b51629': {
name: 'The Language Hackers',
value: true,
},
'283171a6-422c-4991-bc78-95b1b5b51629': {
name: 'The Bug Hackers',
value: false,
},
},
});
const xml = xmlMarshaller.marshallUserData(userData);
expect(xml).to.equal(`<user>
<inbox/>
<newMessages>
<id>283171a5-422c-4991-bc78-95b1b5b51629</id>
<name>The Language Hackers</name>
<value>true</value>
</newMessages>
<newMessages>
<id>283171a6-422c-4991-bc78-95b1b5b51629</id>
<name>The Bug Hackers</name>
<value>false</value>
</newMessages>
</user>`);
});
});
website/server/controllers/top-level/dataexport.js
+5 -35
View File
@@ -1,14 +1,12 @@
import _ from 'lodash';
import moment from 'moment';
import * as js2xml from 'js2xmlparser';
// import Pageres from 'pageres';
// import nconf from 'nconf';
// import got from 'got';
import md from 'habitica-markdown';
import csvStringify from '../../libs/csvStringify';
import {
NotFound,
} from '../../libs/errors';
import { marshallUserData } from '../../libs/xmlMarshaller';
import { NotFound } from '../../libs/errors';
import * as Tasks from '../../models/task';
import * as inboxLib from '../../libs/inbox';
// import { model as User } from '../../models/user';
@@ -85,7 +83,7 @@ api.exportUserHistory = {
// Convert user to json and attach tasks divided by type and inbox messages
// at user.tasks[`${taskType}s`] (user.tasks.{dailys/habits/...})
async function _getUserDataForExport (user, xmlMode = false) {
async function _getUserDataForExport (user) {
const userData = user.toJSON();
userData.tasks = {};
@@ -108,29 +106,6 @@ async function _getUserDataForExport (user, xmlMode = false) {
userData.tasks[`${taskType}s`] = tasksPerType;
});
if (xmlMode) {
// object maps cant be parsed
userData.inbox.messages = _(userData.inbox.messages)
.map(m => {
const flags = Object.keys(m.flags);
m.flags = flags;
return m;
})
.value();
// _id gets parsed as an bytearray => which gets casted to a chararray => "weird chars"
userData.unpinnedItems = userData.unpinnedItems.map(i => ({
path: i.path,
type: i.type,
}));
userData.pinnedItems = userData.pinnedItems.map(i => ({
path: i.path,
type: i.type,
}));
}
return userData;
}
@@ -171,13 +146,8 @@ api.exportUserDataXml = {
url: '/export/userdata.xml',
middlewares: [authWithSession],
async handler (req, res) {
const userData = await _getUserDataForExport(res.locals.user, true);
const xmlData = js2xml.parse('user', userData, {
cdataInvalidChars: true,
declaration: {
include: false,
},
});
const userData = await _getUserDataForExport(res.locals.user);
const xmlData = marshallUserData(userData);
res.set({
'Content-Type': 'text/xml',
website/server/libs/xmlMarshaller.js
+32
View File
@@ -0,0 +1,32 @@
import _ from 'lodash';
import * as js2xml from 'js2xmlparser';
export function marshallUserData (userData) {
// object maps can't be marshalled to XML
userData.inbox.messages = _(userData.inbox.messages)
.map(m => {
m.flags = Object.keys(m.flags);
return m;
})
.value();
userData.newMessages = _.map(userData.newMessages, (msg, id) => ({ id, ...msg }));
// _id gets parsed as a bytearray => which gets cast to a chararray => "weird chars"
userData.unpinnedItems = userData.unpinnedItems.map(i => ({
path: i.path,
type: i.type,
}));
userData.pinnedItems = userData.pinnedItems.map(i => ({
path: i.path,
type: i.type,
}));
return js2xml.parse('user', userData, {
cdataInvalidChars: true,
declaration: {
include: false,
},
});
}
website/server/models/user/schema.js
+1 -3
View File
@@ -1,4 +1,4 @@
import mongoose from 'mongoose';
import { Schema } from 'mongoose';
import validator from 'validator';
import shared from '../../../common';
import { // eslint-disable-line import/no-cycle
@@ -10,8 +10,6 @@ import { schema as TagSchema } from '../tag';
import { schema as UserNotificationSchema } from '../userNotification';
import { schema as WebhookSchema } from '../webhook';
const { Schema } = mongoose;
const RESTRICTED_EMAIL_DOMAINS = Object.freeze(['habitica.com', 'habitrpg.com']);
// User schema definition