Kai/habitica
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

API v3 Rate Limiter (#12117)

Browse Source 
* simplify ip address management by using the trust proxy express option

* add setupExpress file

* fix redirects middleware tests

* fix lint

* short circuit the ip blocking middleware

* basic implementation with ip based limiting

* improve logging

* upgrade apidoc

* apidoc: add introduction section

* fix lint

* fix tests

* fix lint

* add unit tests for rate limiter

* do not send retry-after header when points are available

* automatically fix lint

* fix more lint issues

* use userId as key for rate limit when available
This commit is contained in:
Matteo Pagliazzi
2020-07-17 16:13:51 +02:00
committed by GitHub
parent 0261d12bd9
commit f1173cee6a
15 changed files with 383 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files
website/server/libs/errors.js
+13
View File
@@ -54,6 +54,19 @@ export const { NotFound } = common.errors;
*/
export const { Forbidden } = common.errors;
/**
* @apiDefine TooManyRequests
* @apiError TooManyRequests The client made too many requests to the API and was rate limited.
*
* @apiErrorExample Error-Response:
* HTTP/1.1 429 TooManyRequests
* {
* "error": "TooManyRequests",
* "message": "Access forbidden."
* }
*/
export const { TooManyRequests } = common.errors;
/**
* @apiDefine NotificationNotFound
* @apiError NotificationNotFound The notification was not found.
website/server/libs/setupExpress.js
+11
View File
@@ -0,0 +1,11 @@
import nconf from 'nconf';
const IS_PROD = nconf.get('IS_PROD');
export default function setupExpress (app) {
app.set('view engine', 'pug');
app.set('views', `${__dirname}/../../views`);
// The production build of Habitica runs behind a proxy
// See https://expressjs.com/it/guide/behind-proxies.html
if (IS_PROD) app.set('trust proxy', true);
}
website/server/middlewares/appRoutes.js
+4 -3
View File
@@ -3,6 +3,8 @@ import expressValidator from 'express-validator';
import path from 'path';
import analytics from './analytics';
import setupBody from './setupBody';
import rateLimiter from './rateLimiter';
import setupExpress from '../libs/setupExpress';
import * as routes from '../libs/routes';
const API_V3_CONTROLLERS_PATH = path.join(__dirname, '/../controllers/api-v3/');
@@ -12,8 +14,7 @@ const TOP_LEVEL_CONTROLLERS_PATH = path.join(__dirname, '/../controllers/top-lev
const app = express();
// re-set the view options because they are not inherited from the top level app
app.set('view engine', 'pug');
app.set('views', `${__dirname}/../../views`);
setupExpress(app);
app.use(expressValidator());
app.use(analytics);
@@ -26,7 +27,7 @@ app.use('/', topLevelRouter);
const v3Router = express.Router(); // eslint-disable-line new-cap
routes.walkControllers(v3Router, API_V3_CONTROLLERS_PATH);
app.use('/api/v3', v3Router);
app.use('/api/v3', rateLimiter, v3Router);
// API v4 proxies API v3 routes by default.
// It can also disable or override v3 routes
website/server/middlewares/index.js
+2 -2
View File
@@ -9,6 +9,7 @@ import methodOverride from 'method-override';
import passport from 'passport';
import basicAuth from 'express-basic-auth';
import helmet from 'helmet';
import setupExpress from '../libs/setupExpress';
import errorHandler from './errorHandler';
import notFoundHandler from './notFound';
import cors from './cors';
@@ -39,8 +40,7 @@ const SESSION_SECRET = nconf.get('SESSION_SECRET');
const TEN_YEARS = 1000 * 60 * 60 * 24 * 365 * 10;
export default function attachMiddlewares (app, server) {
app.set('view engine', 'pug');
app.set('views', `${__dirname}/../../views`);
setupExpress(app);
app.use(domainMiddleware(server, mongoose));
website/server/middlewares/ipBlocker.js
+2 -24
View File
@@ -26,30 +26,8 @@ export default function ipBlocker (req, res, next) {
// If there are no IPs to block, skip the middleware
if (blockedIps.length === 0) return next();
// If x-forwarded-for is undefined we're not behind the production proxy
const originIpsRaw = req.header('x-forwarded-for');
if (!originIpsRaw) return next();
// Format xxx.xxx.xxx.xxx, xxx.xxx.xxx.xxx (comma separated list of ip)
const originIps = originIpsRaw
.split(',')
.map(originIp => originIp.trim());
// We try to match any of the origins IPs against the blocked IPs list.
//
// In case we're behind a Google Cloud Load Balancer the last ip
// in the list is added by the load balancer.
// See https://cloud.google.com/load-balancing/docs/https#target-proxies
// In particular:
// << A Google Cloud external HTTP(S) load balancer adds two IP addresses to the header:
// the IP address of the requesting client and the external IP address of the load balancer's
// forwarding rule, in that order.
// Therefore, the IP address that immediately precedes the Google Cloud load balancer's
// IP address is the IP address of the system that contacts the load balancer.
// The system might be a client, or it might be another proxy server, outside Google Cloud,
// that forwards requests on behalf of a client. >>
const match = originIps.find(originIp => blockedIps.includes(originIp)) !== undefined;
// Is the client IP, req.ip, blocked?
const match = blockedIps.find(blockedIp => blockedIp === req.ip) !== undefined;
if (match === true) {
// Not translated because no user is loaded at this point
website/server/middlewares/rateLimiter.js
+94
View File
@@ -0,0 +1,94 @@
import nconf from 'nconf';
import redis from 'redis';
import {
RateLimiterRedis,
RateLimiterMemory,
RateLimiterRes,
} from 'rate-limiter-flexible';
import {
TooManyRequests,
} from '../libs/errors';
import logger from '../libs/logger';
import apiError from '../libs/apiError';
// Middleware to rate limit requests to the API
// More info on the API rate limits can be found on the wiki at
// https://habitica.fandom.com/wiki/Guidance_for_Comrades#Rules_for_Third-Party_Tools
const IS_TEST = nconf.get('IS_TEST');
const RATE_LIMITER_ENABLED = nconf.get('RATE_LIMITER_ENABLED') === 'true';
const REDIS_HOST = nconf.get('REDIS_HOST');
const REDIS_PASSWORD = nconf.get('REDIS_PASSWORD');
const REDIS_PORT = nconf.get('REDIS_PORT');
let redisClient;
let rateLimiter;
const rateLimiterOpts = {
keyPrefix: 'api-v3',
points: 30, // 30 requests
duration: 60, // per 1 minute by User ID or IP
};
if (RATE_LIMITER_ENABLED) {
if (IS_TEST) {
rateLimiter = new RateLimiterMemory({
...rateLimiterOpts,
});
} else {
redisClient = redis.createClient({
host: REDIS_HOST,
password: REDIS_PASSWORD,
port: REDIS_PORT,
enable_offline_queue: false,
});
redisClient.on('error', error => {
logger.error(error, 'Redis Error');
});
rateLimiter = new RateLimiterRedis({
...rateLimiterOpts,
storeClient: redisClient,
});
}
}
function setResponseHeaders (res, rateLimiterRes) {
const headers = {
'X-RateLimit-Limit': rateLimiterOpts.points,
'X-RateLimit-Remaining': rateLimiterRes.remainingPoints,
'X-RateLimit-Reset': new Date(Date.now() + rateLimiterRes.msBeforeNext),
};
if (rateLimiterRes.remainingPoints < 1) {
headers['Retry-After'] = rateLimiterRes.msBeforeNext / 1000;
}
res.set(headers);
}
export default function rateLimiterMiddleware (req, res, next) {
if (!RATE_LIMITER_ENABLED) return next();
const userId = req.header('x-api-user');
return rateLimiter.consume(userId || req.ip)
.then(rateLimiterRes => {
setResponseHeaders(res, rateLimiterRes);
return next();
})
.catch(rateLimiterRes => {
if (rateLimiterRes instanceof RateLimiterRes) {
setResponseHeaders(res, rateLimiterRes);
return next(new TooManyRequests(apiError('clientRateLimited')));
}
// In case of an unhandled error we skip the middleware as it could mean
// , for example, that the connection to the redis database is not working.
// We do not want to block all requests in these cases.
logger.error(rateLimiterRes, 'Rate Limiter Error');
return next();
});
}
website/server/middlewares/redirects.js
+4 -3
View File
@@ -4,6 +4,8 @@ import url from 'url';
const IS_PROD = nconf.get('IS_PROD');
const IGNORE_REDIRECT = nconf.get('IGNORE_REDIRECT') === 'true';
const BASE_URL = nconf.get('BASE_URL');
const HTTPS_BASE_URL = BASE_URL.indexOf('https') === 0;
// A secret key that if passed as req.query.skipSSLCheck allows to skip
// the redirects to SSL, used for health checks from the load balancer
const SKIP_SSL_CHECK_KEY = nconf.get('SKIP_SSL_CHECK_KEY');
@@ -12,10 +14,9 @@ const BASE_URL_HOST = url.parse(BASE_URL).hostname;
function isHTTP (req) {
return ( // eslint-disable-line no-extra-parens
req.header('x-forwarded-proto')
&& req.header('x-forwarded-proto') === 'http'
req.protocol === 'http'
&& IS_PROD
&& BASE_URL.indexOf('https') === 0
&& HTTPS_BASE_URL === true
);
}