From f59e364a857e2f361e2e420229615ef7b2866fd7 Mon Sep 17 00:00:00 2001 From: MeiMei <30769358+mei23@users.noreply.github.com> Date: Tue, 31 May 2022 17:44:22 +0900 Subject: [PATCH 1/8] Fix IP address rate limit (#8758) * Fix IP address rate limit * CHANGELOG * Tune getIpHash --- CHANGELOG.md | 2 +- packages/backend/src/misc/get-ip-hash.ts | 9 +++++++++ packages/backend/src/server/api/call.ts | 11 +++-------- packages/backend/src/server/api/private/signin.ts | 3 ++- 4 files changed, 15 insertions(+), 10 deletions(-) create mode 100644 packages/backend/src/misc/get-ip-hash.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b086ddba..cfcf52ce9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,7 @@ You should also include the user name that made the change. Your own theme color may be unset if it was in an invalid format. Admins should check their instance settings if in doubt. - Perform port diagnosis at startup only when Listen fails @mei23 -- Rate limiting is now also usable for non-authenticated users. @Johann150 +- Rate limiting is now also usable for non-authenticated users. @Johann150 @mei23 Admins should make sure the reverse proxy sets the `X-Forwarded-For` header to the original address. ### Bugfixes diff --git a/packages/backend/src/misc/get-ip-hash.ts b/packages/backend/src/misc/get-ip-hash.ts new file mode 100644 index 000000000..379325bb1 --- /dev/null +++ b/packages/backend/src/misc/get-ip-hash.ts @@ -0,0 +1,9 @@ +import IPCIDR from 'ip-cidr'; + +export function getIpHash(ip: string) { + // because a single person may control many IPv6 addresses, + // only a /64 subnet prefix of any IP will be taken into account. + // (this means for IPv4 the entire address is used) + const prefix = IPCIDR.createAddress(ip).mask(64); + return 'ip-' + BigInt('0b' + prefix).toString(36); +} diff --git a/packages/backend/src/server/api/call.ts b/packages/backend/src/server/api/call.ts index fbe25e173..cd3e0abc0 100644 --- a/packages/backend/src/server/api/call.ts +++ b/packages/backend/src/server/api/call.ts @@ -6,7 +6,7 @@ import endpoints, { IEndpointMeta } from './endpoints.js'; import { ApiError } from './error.js'; import { apiLogger } from './logger.js'; import { AccessToken } from '@/models/entities/access-token.js'; -import IPCIDR from 'ip-cidr'; +import { getIpHash } from '@/misc/get-ip-hash.js'; const accessDenied = { message: 'Access denied.', @@ -33,18 +33,13 @@ export default async (endpoint: string, user: CacheableLocalUser | null | undefi throw new ApiError(accessDenied); } - if (ep.meta.requireCredential && ep.meta.limit && !isModerator) { + if (ep.meta.limit && !isModerator) { // koa will automatically load the `X-Forwarded-For` header if `proxy: true` is configured in the app. let limitActor: string; if (user) { limitActor = user.id; } else { - // because a single person may control many IPv6 addresses, - // only a /64 subnet prefix of any IP will be taken into account. - // (this means for IPv4 the entire address is used) - const ip = IPCIDR.createAddress(ctx.ip).mask(64); - - limitActor = 'ip-' + parseInt(ip, 2).toString(36); + limitActor = getIpHash(ctx!.ip); } const limit = Object.assign({}, ep.meta.limit); diff --git a/packages/backend/src/server/api/private/signin.ts b/packages/backend/src/server/api/private/signin.ts index b304550e2..79b31764f 100644 --- a/packages/backend/src/server/api/private/signin.ts +++ b/packages/backend/src/server/api/private/signin.ts @@ -10,6 +10,7 @@ import { verifyLogin, hash } from '../2fa.js'; import { randomBytes } from 'node:crypto'; import { IsNull } from 'typeorm'; import { limiter } from '../limiter.js'; +import { getIpHash } from '@/misc/get-ip-hash.js'; export default async (ctx: Koa.Context) => { ctx.set('Access-Control-Allow-Origin', config.url); @@ -27,7 +28,7 @@ export default async (ctx: Koa.Context) => { try { // not more than 1 attempt per second and not more than 10 attempts per hour - await limiter({ key: 'signin', duration: 60 * 60 * 1000, max: 10, minInterval: 1000 }, ctx.ip); + await limiter({ key: 'signin', duration: 60 * 60 * 1000, max: 10, minInterval: 1000 }, getIpHash(ctx.ip)); } catch (err) { ctx.status = 429; ctx.body = { From 8e296b239875775794aeb84c8f59ade25a09bbc2 Mon Sep 17 00:00:00 2001 From: Johann150 Date: Tue, 31 May 2022 10:54:02 +0200 Subject: [PATCH 2/8] fix: always remove completed tasks (#8771) --- packages/backend/src/queue/index.ts | 2 ++ packages/backend/src/services/note/create.ts | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/queue/index.ts b/packages/backend/src/queue/index.ts index 67d5f5d24..c5fd7de1c 100644 --- a/packages/backend/src/queue/index.ts +++ b/packages/backend/src/queue/index.ts @@ -305,11 +305,13 @@ export default function() { systemQueue.add('resyncCharts', { }, { repeat: { cron: '0 0 * * *' }, + removeOnComplete: true, }); systemQueue.add('cleanCharts', { }, { repeat: { cron: '0 0 * * *' }, + removeOnComplete: true, }); systemQueue.add('checkExpiredMutings', { diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index ceb5e8cc7..e2bf9d5b5 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -312,7 +312,8 @@ export default async (user: { id: User['id']; username: User['username']; host: endedPollNotificationQueue.add({ noteId: note.id, }, { - delay + delay, + removeOnComplete: true, }); } From 857b9cab2b7d7a97cd3210a09b1889c0e3cc1239 Mon Sep 17 00:00:00 2001 From: MeiMei <30769358+mei23@users.noreply.github.com> Date: Tue, 31 May 2022 17:55:07 +0900 Subject: [PATCH 3/8] Fix `Cannot find module` issue (#8770) * Add --force to yarn in the installation script * CHAGELOG --- CHANGELOG.md | 1 + scripts/install-packages.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfcf52ce9..968759501 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ You should also include the user name that made the change. - Server: use correct order of attachments on notes @Johann150 - Server: prevent crash when processing certain PNGs @syuilo - Server: Fix unable to generate video thumbnails @mei23 +- Server: Fix `Cannot find module` issue @mei23 ## 12.110.1 (2022/04/23) diff --git a/scripts/install-packages.js b/scripts/install-packages.js index bc8e016a3..d1dea3ebe 100644 --- a/scripts/install-packages.js +++ b/scripts/install-packages.js @@ -3,7 +3,7 @@ const execa = require('execa'); (async () => { console.log('installing dependencies of packages/backend ...'); - await execa('yarn', ['install'], { + await execa('yarn', ['--force', 'install'], { cwd: __dirname + '/../packages/backend', stdout: process.stdout, stderr: process.stderr, From f0b27fa22b50e3c989aae500e973efec028c15bd Mon Sep 17 00:00:00 2001 From: Andreas Nedbal Date: Tue, 31 May 2022 10:57:01 +0200 Subject: [PATCH 4/8] Extract commonly used test logic to commands (#8767) * meta(tests): enable workflows to run in branch * feat(tests): move commonly used logic to Cypress commands * chore(tests): replace more code with commands * meta(tests): disable workflows to run in branch --- cypress/integration/basic.js | 63 ++++++---------------------------- cypress/integration/widgets.js | 27 +++------------ cypress/support/commands.js | 30 ++++++++++++++++ 3 files changed, 44 insertions(+), 76 deletions(-) diff --git a/cypress/integration/basic.js b/cypress/integration/basic.js index eb15cfe22..eb5195c4b 100644 --- a/cypress/integration/basic.js +++ b/cypress/integration/basic.js @@ -1,11 +1,6 @@ describe('Before setup instance', () => { beforeEach(() => { - cy.window(win => { - win.indexedDB.deleteDatabase('keyval-store'); - }); - cy.request('POST', '/api/reset-db').as('reset'); - cy.get('@reset').its('status').should('equal', 204); - cy.reload(true); + cy.resetState(); }); afterEach(() => { @@ -35,18 +30,10 @@ describe('Before setup instance', () => { describe('After setup instance', () => { beforeEach(() => { - cy.window(win => { - win.indexedDB.deleteDatabase('keyval-store'); - }); - cy.request('POST', '/api/reset-db').as('reset'); - cy.get('@reset').its('status').should('equal', 204); - cy.reload(true); + cy.resetState(); // インスタンス初期セットアップ - cy.request('POST', '/api/admin/accounts/create', { - username: 'admin', - password: 'pass', - }).its('body').as('admin'); + cy.registerUser('admin', 'pass', true); }); afterEach(() => { @@ -76,24 +63,13 @@ describe('After setup instance', () => { describe('After user signup', () => { beforeEach(() => { - cy.window(win => { - win.indexedDB.deleteDatabase('keyval-store'); - }); - cy.request('POST', '/api/reset-db').as('reset'); - cy.get('@reset').its('status').should('equal', 204); - cy.reload(true); + cy.resetState(); // インスタンス初期セットアップ - cy.request('POST', '/api/admin/accounts/create', { - username: 'admin', - password: 'pass', - }).its('body').as('admin'); + cy.registerUser('admin', 'pass', true); // ユーザー作成 - cy.request('POST', '/api/signup', { - username: 'alice', - password: 'alice1234', - }).its('body').as('alice'); + cy.registerUser('alice', 'alice1234'); }); afterEach(() => { @@ -138,34 +114,15 @@ describe('After user signup', () => { describe('After user singed in', () => { beforeEach(() => { - cy.window(win => { - win.indexedDB.deleteDatabase('keyval-store'); - }); - cy.request('POST', '/api/reset-db').as('reset'); - cy.get('@reset').its('status').should('equal', 204); - cy.reload(true); + cy.resetState(); // インスタンス初期セットアップ - cy.request('POST', '/api/admin/accounts/create', { - username: 'admin', - password: 'pass', - }).its('body').as('admin'); + cy.registerUser('admin', 'pass', true); // ユーザー作成 - cy.request('POST', '/api/signup', { - username: 'alice', - password: 'alice1234', - }).its('body').as('alice'); + cy.registerUser('alice', 'alice1234'); - cy.visit('/'); - - cy.intercept('POST', '/api/signin').as('signin'); - - cy.get('[data-cy-signin]').click(); - cy.get('[data-cy-signin-username] input').type('alice'); - cy.get('[data-cy-signin-password] input').type('alice1234{enter}'); - - cy.wait('@signin').as('signedIn'); + cy.login('alice', 'alice1234'); }); afterEach(() => { diff --git a/cypress/integration/widgets.js b/cypress/integration/widgets.js index d63ff274b..56ad95ee9 100644 --- a/cypress/integration/widgets.js +++ b/cypress/integration/widgets.js @@ -1,34 +1,15 @@ describe('After user signed in', () => { beforeEach(() => { - cy.window(win => { - win.indexedDB.deleteDatabase('keyval-store'); - }); + cy.resetState(); cy.viewport('macbook-16'); - cy.request('POST', '/api/reset-db').as('reset'); - cy.get('@reset').its('status').should('equal', 204); - cy.reload(true); // インスタンス初期セットアップ - cy.request('POST', '/api/admin/accounts/create', { - username: 'admin', - password: 'pass', - }).its('body').as('admin'); + cy.registerUser('admin', 'pass', true); // ユーザー作成 - cy.request('POST', '/api/signup', { - username: 'alice', - password: 'alice1234', - }).its('body').as('alice'); + cy.registerUser('alice', 'alice1234'); - cy.visit('/'); - - cy.intercept('POST', '/api/signin').as('signin'); - - cy.get('[data-cy-signin]').click(); - cy.get('[data-cy-signin-username] input').type('alice'); - cy.get('[data-cy-signin-password] input').type('alice1234{enter}'); - - cy.wait('@signin').as('signedIn'); + cy.login('alice', 'alice1234'); }); afterEach(() => { diff --git a/cypress/support/commands.js b/cypress/support/commands.js index 119ab03f7..95bfcf685 100644 --- a/cypress/support/commands.js +++ b/cypress/support/commands.js @@ -23,3 +23,33 @@ // // -- This will overwrite an existing command -- // Cypress.Commands.overwrite('visit', (originalFn, url, options) => { ... }) + +Cypress.Commands.add('resetState', () => { + cy.window(win => { + win.indexedDB.deleteDatabase('keyval-store'); + }); + cy.request('POST', '/api/reset-db').as('reset'); + cy.get('@reset').its('status').should('equal', 204); + cy.reload(true); +}); + +Cypress.Commands.add('registerUser', (username, password, isAdmin = false) => { + const route = isAdmin ? '/api/admin/accounts/create' : '/api/signup'; + + cy.request('POST', route, { + username: username, + password: password, + }).its('body').as(username); +}); + +Cypress.Commands.add('login', (username, password) => { + cy.visit('/'); + + cy.intercept('POST', '/api/signin').as('signin'); + + cy.get('[data-cy-signin]').click(); + cy.get('[data-cy-signin-username] input').type(username); + cy.get('[data-cy-signin-password] input').type(`${password}{enter}`); + + cy.wait('@signin').as('signedIn'); +}); From 7c9d07cd536c847b7e1532d31dfa90450402710d Mon Sep 17 00:00:00 2001 From: Johann150 Date: Tue, 31 May 2022 11:57:55 +0200 Subject: [PATCH 5/8] fix(mfm): remove duplicate br tag/newline (#8616) --- packages/backend/src/mfm/from-html.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/backend/src/mfm/from-html.ts b/packages/backend/src/mfm/from-html.ts index 623cb0e71..15110b6b7 100644 --- a/packages/backend/src/mfm/from-html.ts +++ b/packages/backend/src/mfm/from-html.ts @@ -6,6 +6,9 @@ const urlRegex = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+/; const urlRegexFull = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+$/; export function fromHtml(html: string, hashtagNames?: string[]): string { + // some AP servers like Pixelfed use br tags as well as newlines + html = html.replace(/\r?\n/gi, '\n'); + const dom = parse5.parseFragment(html); let text = ''; From d3e65bc8b7efe485d19703c6b285c45f828bbc67 Mon Sep 17 00:00:00 2001 From: Johann150 Date: Tue, 31 May 2022 16:22:00 +0200 Subject: [PATCH 6/8] fix(lint): indentation --- packages/client/src/ui/deck.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/ui/deck.vue b/packages/client/src/ui/deck.vue index 1e0d9a165..e538a93f0 100644 --- a/packages/client/src/ui/deck.vue +++ b/packages/client/src/ui/deck.vue @@ -17,7 +17,7 @@ :key="ids[0]" class="column" :column="columns.find(c => c.id === ids[0])" - :is-stacked="false" + :is-stacked="false" :style="columns.find(c => c.id === ids[0])!.flexible ? { flex: 1, minWidth: '350px' } : { width: columns.find(c => c.id === ids[0])!.width + 'px' }" @parent-focus="moveFocus(ids[0], $event)" /> From 58752fab0b84d7e322c5c3c13594e497cd5b951b Mon Sep 17 00:00:00 2001 From: Johann150 Date: Wed, 1 Jun 2022 08:51:00 +0200 Subject: [PATCH 7/8] fix: server metrics widget --- packages/client/src/widgets/server-metric/net.vue | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/client/src/widgets/server-metric/net.vue b/packages/client/src/widgets/server-metric/net.vue index 82b3a67d7..b698953f9 100644 --- a/packages/client/src/widgets/server-metric/net.vue +++ b/packages/client/src/widgets/server-metric/net.vue @@ -94,10 +94,10 @@ function onStats(connStats) { inPolygonPoints = `${viewBoxX - (stats.length - 1)},${viewBoxY} ${inPolylinePoints} ${viewBoxX},${viewBoxY}`; outPolygonPoints = `${viewBoxX - (stats.length - 1)},${viewBoxY} ${outPolylinePoints} ${viewBoxX},${viewBoxY}`; - inHeadX = inPolylinePoints[inPolylinePoints.length - 1][0]; - inHeadY = inPolylinePoints[inPolylinePoints.length - 1][1]; - outHeadX = outPolylinePoints[outPolylinePoints.length - 1][0]; - outHeadY = outPolylinePoints[outPolylinePoints.length - 1][1]; + inHeadX = inPolylinePointsStats[inPolylinePointsStats.length - 1][0]; + inHeadY = inPolylinePointsStats[inPolylinePointsStats.length - 1][1]; + outHeadX = outPolylinePointsStats[outPolylinePointsStats.length - 1][0]; + outHeadY = outPolylinePointsStats[outPolylinePointsStats.length - 1][1]; inRecent = connStats.net.rx; outRecent = connStats.net.tx; From 54470123524a4c5841811fbd8b15376d7dfda6e5 Mon Sep 17 00:00:00 2001 From: Johann150 Date: Wed, 1 Jun 2022 09:34:40 +0200 Subject: [PATCH 8/8] fix(dev): no labels for l10n_develop --- .github/workflows/labeler.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 057208eda..fa4a58c3a 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -1,6 +1,8 @@ name: "Pull Request Labeler" on: -- pull_request_target + pull_request_target: + branches-ignore: + - 'l10n_develop' jobs: triage: