From a6db8e693ff23c968d90b29c4750956da6832480 Mon Sep 17 00:00:00 2001 From: Matteo Pagliazzi Date: Fri, 19 Feb 2016 12:13:48 +0100 Subject: [PATCH] refactor dataexport controller --- package.json | 6 +- .../controllers/{ => api-v2}/dataexport.js | 15 -- website/src/controllers/api-v3/dataexport.js | 225 ++++++++++++++++++ website/src/libs/api-v3/webhook.js | 2 +- website/src/routes/dataexport.js | 2 +- 5 files changed, 230 insertions(+), 20 deletions(-) rename website/src/controllers/{ => api-v2}/dataexport.js (91%) create mode 100644 website/src/controllers/api-v3/dataexport.js diff --git a/package.json b/package.json index a068f199e1..dc268a8f77 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "firebase": "^2.2.9", "firebase-token-generator": "^2.0.0", "glob": "^4.3.5", + "got": "^6.1.1", "grunt": "~0.4.1", "grunt-cli": "~0.1.9", "grunt-contrib-clean": "~0.6.0", @@ -52,7 +53,7 @@ "image-size": "~0.3.2", "in-app-purchase": "^0.2.0", "jade": "~1.11.0", - "js2xmlparser": "~0.1.2", + "js2xmlparser": "~1.0.0", "lodash": "^3.10.1", "loggly": "~1.0.8", "marked": "^0.3.5", @@ -67,7 +68,7 @@ "nib": "~1.0.1", "nodemailer": "^1.9.0", "object-path": "^0.9.2", - "pageres": "^1.0.1", + "pageres": "^4.1.0", "passport": "~0.2.1", "passport-facebook": "2.0.0", "paypal-ipn": "2.1.0", @@ -77,7 +78,6 @@ "push-notify": "^1.1.1", "q": "^1.4.1", "request": "~2.44.0", - "s3-upload-stream": "^1.0.6", "serve-favicon": "^2.3.0", "stripe": "*", "superagent": "~1.4.0", diff --git a/website/src/controllers/dataexport.js b/website/src/controllers/api-v2/dataexport.js similarity index 91% rename from website/src/controllers/dataexport.js rename to website/src/controllers/api-v2/dataexport.js index 1c88d2d90f..ad8a998dd2 100644 --- a/website/src/controllers/dataexport.js +++ b/website/src/controllers/api-v2/dataexport.js @@ -22,21 +22,6 @@ var request = require('request'); ------------------------------------------------------------------------ */ -dataexport.history = function(req, res) { - var user = res.locals.user; - var output = [ - ["Task Name", "Task ID", "Task Type", "Date", "Value"] - ]; - _.each(user.tasks, function(task) { - _.each(task.history, function(history) { - output.push( - [task.text, task.id, task.type, moment(history.date).format("MM-DD-YYYY HH:mm:ss"), history.value] - ); - }); - }); - return res.csv(output); -} - var userdata = function(user) { if(user.auth && user.auth.local) { delete user.auth.local.salt; diff --git a/website/src/controllers/api-v3/dataexport.js b/website/src/controllers/api-v3/dataexport.js new file mode 100644 index 0000000000..3fd3399ff2 --- /dev/null +++ b/website/src/controllers/api-v3/dataexport.js @@ -0,0 +1,225 @@ +import { authWithSession } from '../../middlewares/api-v3/auth'; +import cron from '../../middlewares/api-v3/cron'; +import { model as User } from '../../models/user'; +import * as Tasks from '../../models/task'; +import { + NotFound, +} from '../../libs/api-v3/errors'; +import _ from 'lodash'; +import csvStringify from '../../libs/api-v3/csvStringify'; +import moment from 'moment'; +import js2xml from 'js2xmlparser'; +import Pageres from 'pageres'; +import AWS from 'aws-sdk'; +import nconf from 'nconf'; +import got from 'got'; +import Q from 'q'; + +let S3 = new AWS.S3({ + accessKeyId: nconf.get('S3:accessKeyId'), + secretAccessKey: nconf.get('S3:secretAccessKey'), +}); +const S3_BUCKET = nconf.get('S3:bucket'); + +const BASE_URL = nconf.get('BASE_URL'); + +let api = {}; + +/** + * @api {get} /export/history.csv Export user tasks history in CSV format. History is only available for habits and dailys so todos and rewards won't be included + * @apiVersion 3.0.0 + * @apiName ExportUserHistory + * @apiGroup DataExport + * + * @apiSuccess {string} A cvs file + */ +api.exportUserHistory = { + method: 'GET', + url: '/export/history.csv', + middlewares: [authWithSession(), cron], + async handler (req, res) { + let user = res.locals.user; + + let tasks = await Tasks.Task.find({ + userId: user._id, + type: {$in: ['habit', 'daily']}, + }).exec(); + + let output = [ + ['Task Name', 'Task ID', 'Task Type', 'Date', 'Value'], + ]; + + tasks.forEach(task => { + task.history.forEach(history => { + output.push([ + task.text, + task._id, + task.type, + moment(history.date).format('YYYY-MM-DD HH:mm:ss'), + history.value, + ]); + }); + }); + + res.set({ + 'Content-Type': 'text/csv', + 'Content-disposition': `attachment; filename=habitica-tasks-history.csv`, + }); + + let csvRes = await csvStringify(output); + res.status(200).send(csvRes); + }, +}; + +// Convert user to json and attach tasks divided by type +// at user.tasks[`${taskType}s`] (user.tasks.{dailys/habits/...}) +async function _getUserDataForExport (user) { + let userData = user.toJSON(); + userData.tasks = {}; + + let tasks = await Tasks.Task.find({ + userId: user._id, + }).exec(); + + tasks = _.chain(tasks) + .map(task => task.toJSON()) + .groupBy(task => task.type) + .each((tasksPerType, taskType) => { + userData.tasks[`${taskType}s`] = tasksPerType; + }) + .value(); + + return userData; +} + +// TODO export tasks too +/** + * @api {get} /export/userdata.json Export user data in JSON format. + * @apiVersion 3.0.0 + * @apiName ExportUserDataJson + * @apiGroup DataExport + * + * @apiSuccess {string} A json file + */ +api.exportUserDataJson = { + method: 'GET', + url: '/export/userdata.json', + middlewares: [authWithSession(), cron], + async handler (req, res) { + let userData = await _getUserDataForExport(res.locals.user); + + res.set({ + 'Content-Type': 'application/json', + 'Content-disposition': `attachment; filename=habitica-user-data.json`, + }); + let jsonRes = JSON.stringify(userData); + + res.status(200).send(jsonRes); + }, +}; + +/** + * @api {get} /export/userdata.xml Export user data in XML format + * @apiVersion 3.0.0 + * @apiName ExportUserDataXml + * @apiGroup DataExport + * + * @apiSuccess {string} A xml file + */ +api.exportUserDataXml = { + method: 'GET', + url: '/export/userdata.xml', + middlewares: [authWithSession(), cron], + async handler (req, res) { + let userData = await _getUserDataForExport(res.locals.user); + + res.set({ + 'Content-Type': 'text/xml', + 'Content-disposition': `attachment; filename=habitica-user-data.xml`, + }); + res.status(200).send(js2xml('user', userData)); + }, +}; + +/** + * @api {get} /export/avatar-:uuid.html Render a user avatar as an HTML page + * @apiVersion 3.0.0 + * @apiName ExportUserAvatarHtml + * @apiGroup DataExport + * + * @apiSuccess {string} An html page + */ +api.exportUserAvatarHtml = { + method: 'GET', + url: '/export/avatar-:memberId.html', + async handler (req, res) { + req.checkParams('memberId', res.t('memberIdRequired')).notEmpty().isUUID(); + + let validationErrors = req.validationErrors(); + if (validationErrors) throw validationErrors; + + let memberId = req.params.memberId; + let member = await User + .findById(memberId) + .select('stats profile items achievements preferences backer contributor') + .exec(); + + if (!member) throw new NotFound(res.t('userWithIDNotFound', {userId: memberId})); + res.render('avatar-static', { + title: member.profile.name, + env: _.defaults({member}, res.locals.habitrpg), // TODO review once static pages are done + }); + }, +}; + +/** + * @api {get} /export/avatar-:uuid.html Export a user avatar as a PNG file + * @apiVersion 3.0.0 + * @apiName ExportUserAvatarPng + * @apiGroup DataExport + * + * @apiSuccess {string} A png file + */ +api.exportUserAvatarPng = { + method: 'GET', + url: '/export/avatar-:memberId.png', + async handler (req, res) { + req.checkParams('memberId', res.t('memberIdRequired')).notEmpty().isUUID(); + + let validationErrors = req.validationErrors(); + if (validationErrors) throw validationErrors; + + let memberId = req.params.memberId; + + let filename = `avatars/${memberId}/.png`; + let s3url = `https://${S3_BUCKET}+'.s3.amazonaws.com/${filename}`; + let response = await got.head(s3url); + + // cache images for 10 minutes on aws, else upload a new one + if (response.statusCode === 200 && moment().diff(response.headers['last-modified'], 'minutes') < 10) { + return res.redirect(301, s3url); + } + + let [stream] = await new Pageres() + .src(`${BASE_URL}/export/avatar-${memberId}.html`, ['140x147'], { + crop: true, + filename: filename.replace('.png', ''), + }) + .run(); + + let s3upload = S3.upload({ + Bucket: S3_BUCKET, + Key: filename, + ACL: 'public-read', + StorageClass: 'REDUCED_REDUNDANCY', + ContentType: 'image/png', + Expires: moment().add({minutes: 3}), + Body: stream, + }); + + let s3res = await Q.ninvoke(s3upload, 'send'); + res.redirect(s3res.Location); + }, +}; + +export default api; diff --git a/website/src/libs/api-v3/webhook.js b/website/src/libs/api-v3/webhook.js index f1c6f13df2..20d814ecb0 100644 --- a/website/src/libs/api-v3/webhook.js +++ b/website/src/libs/api-v3/webhook.js @@ -7,7 +7,7 @@ let _sendWebhook = (url, body) => { url, body, json: true, - }); + }); // TODO use promises and handle errors }; let _isInvalidWebhook = (hook) => { diff --git a/website/src/routes/dataexport.js b/website/src/routes/dataexport.js index d7328434a0..b267005d24 100644 --- a/website/src/routes/dataexport.js +++ b/website/src/routes/dataexport.js @@ -1,6 +1,6 @@ var express = require('express'); var router = new express.Router(); -var dataexport = require('../controllers/dataexport'); +var dataexport = require('../controllers/api-v2/dataexport'); var auth = require('../controllers/api-v2/auth'); var nconf = require('nconf'); var i18n = require('../libs/api-v2/i18n');