From d84e0265e5250342a82299c7d117c0545c9beb16 Mon Sep 17 00:00:00 2001
From: Akihiko Odaki <nekomanma@pixiv.co.jp>
Date: Sat, 31 Mar 2018 19:55:00 +0900
Subject: [PATCH] Implement remote status retrieval

---
 package.json                                  |   1 +
 src/{server/api => }/common/drive/add-file.ts |  12 +-
 .../api => }/common/drive/upload_from_url.ts  |   2 +-
 src/{server/api => common}/event.ts           |   4 +-
 src/{server/api => }/common/push-sw.ts        |   4 +-
 src/common/remote/activitypub/act/create.ts   |   9 +
 src/common/remote/activitypub/act/index.ts    |  22 +++
 src/common/remote/activitypub/create.ts       |  86 ++++++++++
 .../remote/activitypub/resolve-person.ts      | 104 +++++++++++
 src/common/remote/activitypub/resolver.ts     |  97 +++++++++++
 src/common/remote/activitypub/type.ts         |   3 +
 src/common/remote/resolve-user.ts             |  26 +++
 src/common/remote/webfinger.ts                |  25 +++
 src/models/remote-user-object.ts              |  15 ++
 src/models/user.ts                            |   3 +
 src/processor/http/index.ts                   |   9 +
 src/processor/http/perform-activitypub.ts     |   6 +
 .../{ => http}/report-github-failure.ts       |   4 +-
 src/processor/index.ts                        |  13 +-
 src/server/api/common/notify.ts               |   2 +-
 .../api/common/read-messaging-message.ts      |   6 +-
 src/server/api/common/read-notification.ts    |   2 +-
 .../api/endpoints/drive/files/create.ts       |   2 +-
 .../api/endpoints/drive/files/update.ts       |   2 +-
 .../endpoints/drive/files/upload_from_url.ts  |   2 +-
 .../api/endpoints/drive/folders/create.ts     |   2 +-
 .../api/endpoints/drive/folders/update.ts     |   2 +-
 src/server/api/endpoints/following/create.ts  |   2 +-
 src/server/api/endpoints/following/delete.ts  |   2 +-
 .../api/endpoints/i/regenerate_token.ts       |   2 +-
 src/server/api/endpoints/i/update.ts          |   2 +-
 .../api/endpoints/i/update_client_setting.ts  |   2 +-
 src/server/api/endpoints/i/update_home.ts     |   2 +-
 .../api/endpoints/i/update_mobile_home.ts     |   2 +-
 .../endpoints/messaging/messages/create.ts    |   4 +-
 .../notifications/mark_as_read_all.ts         |   2 +-
 src/server/api/endpoints/othello/match.ts     |   2 +-
 src/server/api/endpoints/posts/create.ts      |   2 +-
 src/server/api/endpoints/posts/polls/vote.ts  |   2 +-
 .../api/endpoints/posts/reactions/create.ts   |   2 +-
 src/server/api/endpoints/users/show.ts        | 162 +-----------------
 src/server/api/private/signin.ts              |   2 +-
 src/server/api/service/github.ts              |   3 +-
 src/server/api/service/twitter.ts             |   2 +-
 src/server/api/stream/othello-game.ts         |   2 +-
 src/server/api/stream/othello.ts              |   2 +-
 46 files changed, 468 insertions(+), 198 deletions(-)
 rename src/{server/api => }/common/drive/add-file.ts (95%)
 rename src/{server/api => }/common/drive/upload_from_url.ts (92%)
 rename src/{server/api => common}/event.ts (97%)
 rename src/{server/api => }/common/push-sw.ts (92%)
 create mode 100644 src/common/remote/activitypub/act/create.ts
 create mode 100644 src/common/remote/activitypub/act/index.ts
 create mode 100644 src/common/remote/activitypub/create.ts
 create mode 100644 src/common/remote/activitypub/resolve-person.ts
 create mode 100644 src/common/remote/activitypub/resolver.ts
 create mode 100644 src/common/remote/activitypub/type.ts
 create mode 100644 src/common/remote/resolve-user.ts
 create mode 100644 src/common/remote/webfinger.ts
 create mode 100644 src/models/remote-user-object.ts
 create mode 100644 src/processor/http/index.ts
 create mode 100644 src/processor/http/perform-activitypub.ts
 rename src/processor/{ => http}/report-github-failure.ts (87%)

diff --git a/package.json b/package.json
index d1f544f86..4275c1c1c 100644
--- a/package.json
+++ b/package.json
@@ -103,6 +103,7 @@
 		"deep-equal": "1.0.1",
 		"deepcopy": "0.6.3",
 		"diskusage": "0.2.4",
+		"dompurify": "^1.0.3",
 		"elasticsearch": "14.2.2",
 		"element-ui": "2.3.2",
 		"emojilib": "2.2.12",
diff --git a/src/server/api/common/drive/add-file.ts b/src/common/drive/add-file.ts
similarity index 95%
rename from src/server/api/common/drive/add-file.ts
rename to src/common/drive/add-file.ts
index 4551f5574..52a7713dd 100644
--- a/src/server/api/common/drive/add-file.ts
+++ b/src/common/drive/add-file.ts
@@ -10,12 +10,12 @@ import * as debug from 'debug';
 import fileType = require('file-type');
 import prominence = require('prominence');
 
-import DriveFile, { getGridFSBucket } from '../../../../models/drive-file';
-import DriveFolder from '../../../../models/drive-folder';
-import { pack } from '../../../../models/drive-file';
-import event, { publishDriveStream } from '../../event';
-import getAcct from '../../../../common/user/get-acct';
-import config from '../../../../conf';
+import DriveFile, { getGridFSBucket } from '../../models/drive-file';
+import DriveFolder from '../../models/drive-folder';
+import { pack } from '../../models/drive-file';
+import event, { publishDriveStream } from '../event';
+import getAcct from '../user/get-acct';
+import config from '../../conf';
 
 const gm = _gm.subClass({
 	imageMagick: true
diff --git a/src/server/api/common/drive/upload_from_url.ts b/src/common/drive/upload_from_url.ts
similarity index 92%
rename from src/server/api/common/drive/upload_from_url.ts
rename to src/common/drive/upload_from_url.ts
index b825e4c53..5dd969593 100644
--- a/src/server/api/common/drive/upload_from_url.ts
+++ b/src/common/drive/upload_from_url.ts
@@ -1,5 +1,5 @@
 import * as URL from 'url';
-import { IDriveFile, validateFileName } from '../../../../models/drive-file';
+import { IDriveFile, validateFileName } from '../../models/drive-file';
 import create from './add-file';
 import * as debug from 'debug';
 import * as tmp from 'tmp';
diff --git a/src/server/api/event.ts b/src/common/event.ts
similarity index 97%
rename from src/server/api/event.ts
rename to src/common/event.ts
index 98bf16113..53520f11c 100644
--- a/src/server/api/event.ts
+++ b/src/common/event.ts
@@ -1,7 +1,7 @@
 import * as mongo from 'mongodb';
 import * as redis from 'redis';
-import swPush from './common/push-sw';
-import config from '../../conf';
+import swPush from './push-sw';
+import config from '../conf';
 
 type ID = string | mongo.ObjectID;
 
diff --git a/src/server/api/common/push-sw.ts b/src/common/push-sw.ts
similarity index 92%
rename from src/server/api/common/push-sw.ts
rename to src/common/push-sw.ts
index 13227af8d..44c328e83 100644
--- a/src/server/api/common/push-sw.ts
+++ b/src/common/push-sw.ts
@@ -1,7 +1,7 @@
 const push = require('web-push');
 import * as mongo from 'mongodb';
-import Subscription from '../../../models/sw-subscription';
-import config from '../../../conf';
+import Subscription from '../models/sw-subscription';
+import config from '../conf';
 
 if (config.sw) {
 	// アプリケーションの連絡先と、サーバーサイドの鍵ペアの情報を登録
diff --git a/src/common/remote/activitypub/act/create.ts b/src/common/remote/activitypub/act/create.ts
new file mode 100644
index 000000000..6c62f7ab9
--- /dev/null
+++ b/src/common/remote/activitypub/act/create.ts
@@ -0,0 +1,9 @@
+import create from '../create';
+
+export default (resolver, actor, activity) => {
+	if ('actor' in activity && actor.account.uri !== activity.actor) {
+		throw new Error;
+	}
+
+	return create(resolver, actor, activity.object);
+};
diff --git a/src/common/remote/activitypub/act/index.ts b/src/common/remote/activitypub/act/index.ts
new file mode 100644
index 000000000..0f4084a61
--- /dev/null
+++ b/src/common/remote/activitypub/act/index.ts
@@ -0,0 +1,22 @@
+import create from './create';
+import createObject from '../create';
+import Resolver from '../resolver';
+
+export default (actor, value) => {
+	return (new Resolver).resolve(value).then(resolved => Promise.all(resolved.map(async asyncResult => {
+		const { resolver, object } = await asyncResult;
+		const created = await (await createObject(resolver, actor, [object]))[0];
+
+		if (created !== null) {
+			return created;
+		}
+
+		switch (object.type) {
+		case 'Create':
+			return create(resolver, actor, object);
+
+		default:
+			return null;
+		}
+	})));
+}
diff --git a/src/common/remote/activitypub/create.ts b/src/common/remote/activitypub/create.ts
new file mode 100644
index 000000000..4aaaeb306
--- /dev/null
+++ b/src/common/remote/activitypub/create.ts
@@ -0,0 +1,86 @@
+import { JSDOM } from 'jsdom';
+import config from '../../../conf';
+import Post from '../../../models/post';
+import RemoteUserObject, { IRemoteUserObject } from '../../../models/remote-user-object';
+import uploadFromUrl from '../../drive/upload_from_url';
+const createDOMPurify = require('dompurify');
+
+function createRemoteUserObject($ref, $id, { id }) {
+	const object = { $ref, $id };
+
+	if (!id) {
+		return { object };
+	}
+
+	return RemoteUserObject.insert({ uri: id, object });
+}
+
+async function createImage(actor, object) {
+	if ('attributedTo' in object && actor.account.uri !== object.attributedTo) {
+		throw new Error;
+	}
+
+	const { _id } = await uploadFromUrl(object.url, actor);
+	return createRemoteUserObject('driveFiles.files', _id, object);
+}
+
+async function createNote(resolver, actor, object) {
+	if ('attributedTo' in object && actor.account.uri !== object.attributedTo) {
+		throw new Error;
+	}
+
+	const mediaIds = 'attachment' in object &&
+		(await Promise.all(await create(resolver, actor, object.attachment)))
+			.filter(media => media !== null && media.object.$ref === 'driveFiles.files')
+			.map(({ object }) => object.$id);
+
+	const { window } = new JSDOM(object.content);
+
+	const { _id } = await Post.insert({
+		channelId: undefined,
+		index: undefined,
+		createdAt: new Date(object.published),
+		mediaIds,
+		replyId: undefined,
+		repostId: undefined,
+		poll: undefined,
+		text: window.document.body.textContent,
+		textHtml: object.content && createDOMPurify(window).sanitize(object.content),
+		userId: actor._id,
+		appId: null,
+		viaMobile: false,
+		geo: undefined
+	});
+
+	// Register to search database
+	if (object.content && config.elasticsearch.enable) {
+		const es = require('../../db/elasticsearch');
+
+		es.index({
+			index: 'misskey',
+			type: 'post',
+			id: _id.toString(),
+			body: {
+				text: window.document.body.textContent
+			}
+		});
+	}
+
+	return createRemoteUserObject('posts', _id, object);
+}
+
+export default async function create(parentResolver, actor, value): Promise<Promise<IRemoteUserObject>[]> {
+	const results = await parentResolver.resolveRemoteUserObjects(value);
+
+	return results.map(asyncResult => asyncResult.then(({ resolver, object }) => {
+		switch (object.type) {
+		case 'Image':
+			return createImage(actor, object);
+
+		case 'Note':
+			return createNote(resolver, actor, object);
+		}
+
+		return null;
+	}));
+};
diff --git a/src/common/remote/activitypub/resolve-person.ts b/src/common/remote/activitypub/resolve-person.ts
new file mode 100644
index 000000000..c7c131b0e
--- /dev/null
+++ b/src/common/remote/activitypub/resolve-person.ts
@@ -0,0 +1,104 @@
+import { JSDOM } from 'jsdom';
+import { toUnicode } from 'punycode';
+import User, { validateUsername, isValidName, isValidDescription } from '../../../models/user';
+import queue from '../../../queue';
+import webFinger from '../webfinger';
+import create from './create';
+import Resolver from './resolver';
+
+async function isCollection(collection) {
+	return ['Collection', 'OrderedCollection'].includes(collection.type);
+}
+
+export default async (value, usernameLower, hostLower, acctLower) => {
+	if (!validateUsername(usernameLower)) {
+		throw new Error;
+	}
+
+	const { resolver, object } = await (new Resolver).resolveOne(value);
+
+	if (
+		object === null ||
+		object.type !== 'Person' ||
+		typeof object.preferredUsername !== 'string' ||
+		object.preferredUsername.toLowerCase() !== usernameLower ||
+		!isValidName(object.name) ||
+		!isValidDescription(object.summary)
+	) {
+		throw new Error;
+	}
+
+	const [followers, following, outbox, finger] = await Promise.all([
+		resolver.resolveOne(object.followers).then(
+			resolved => isCollection(resolved.object) ? resolved.object : null,
+			() => null
+		),
+		resolver.resolveOne(object.following).then(
+			resolved => isCollection(resolved.object) ? resolved.object : null,
+			() => null
+		),
+		resolver.resolveOne(object.outbox).then(
+			resolved => isCollection(resolved.object) ? resolved.object : null,
+			() => null
+		),
+		webFinger(object.id, acctLower),
+	]);
+
+	const summaryDOM = JSDOM.fragment(object.summary);
+
+	// Create user
+	const user = await User.insert({
+		avatarId: null,
+		bannerId: null,
+		createdAt: Date.parse(object.published),
+		description: summaryDOM.textContent,
+		followersCount: followers.totalItem,
+		followingCount: following.totalItem,
+		name: object.name,
+		postsCount: outbox.totalItem,
+		driveCapacity: 1024 * 1024 * 8, // 8MiB
+		username: object.preferredUsername,
+		usernameLower,
+		host: toUnicode(finger.subject.replace(/^.*?@/, '')),
+		hostLower,
+		account: {
+			uri: object.id,
+		},
+	});
+
+	queue.create('http', {
+		type: 'performActivityPub',
+		actor: user._id,
+		outbox
+	}).save();
+
+	const [avatarId, bannerId] = await Promise.all([
+		object.icon,
+		object.image
+	].map(async value => {
+		if (value === undefined) {
+			return null;
+		}
+
+		try {
+			const created = await create(resolver, user, value);
+
+			await Promise.all(created.map(asyncCreated => asyncCreated.then(created => {
+				if (created !== null && created.object.$ref === 'driveFiles.files') {
+					throw created.object.$id;
+				}
+			}, () => {})));
+
+			return null;
+		} catch (id) {
+			return id;
+		}
+	}));
+
+	User.update({ _id: user._id }, { $set: { avatarId, bannerId } });
+
+	user.avatarId = avatarId;
+	user.bannerId = bannerId;
+
+	return user;
+};
diff --git a/src/common/remote/activitypub/resolver.ts b/src/common/remote/activitypub/resolver.ts
new file mode 100644
index 000000000..50ac1b0b1
--- /dev/null
+++ b/src/common/remote/activitypub/resolver.ts
@@ -0,0 +1,97 @@
+import RemoteUserObject from '../../../models/remote-user-object';
+import { IObject } from './type';
+const request = require('request-promise-native');
+
+type IResult = {
+  resolver: Resolver;
+  object: IObject;
+};
+
+async function resolveUnrequestedOne(this: Resolver, value) {
+	if (typeof value !== 'string') {
+		return { resolver: this, object: value };
+	}
+
+	const resolver = new Resolver(this.requesting);
+
+	resolver.requesting.add(value);
+
+	const object = await request({
+		url: value,
+		headers: {
+			Accept: 'application/activity+json, application/ld+json'
+		},
+		json: true
+	});
+
+	if (object === null || (
+		Array.isArray(object['@context']) ?
+			!object['@context'].includes('https://www.w3.org/ns/activitystreams') :
+			object['@context'] !== 'https://www.w3.org/ns/activitystreams'
+	)) {
+		throw new Error;
+	}
+
+	return { resolver, object };
+}
+
+async function resolveCollection(this: Resolver, value) {
+	if (Array.isArray(value)) {
+		return value;
+	}
+
+	const resolved = typeof value === 'string' ?
+		await resolveUnrequestedOne.call(this, value) :
+		value;
+
+	switch (resolved.type) {
+	case 'Collection':
+		return resolved.items;
+
+	case 'OrderedCollection':
+		return resolved.orderedItems;
+
+	default:
+		return [resolved];
+	}
+}
+
+export default class Resolver {
+	requesting: Set<string>;
+
+	constructor(iterable?: Iterable<string>) {
+		this.requesting = new Set(iterable);
+	}
+
+	async resolve(value): Promise<Promise<IResult>[]> {
+		const collection = await resolveCollection.call(this, value);
+
+		return collection
+			.filter(element => !this.requesting.has(element))
+			.map(resolveUnrequestedOne.bind(this));
+	}
+
+	resolveOne(value) {
+		if (this.requesting.has(value)) {
+			throw new Error;
+		}
+
+		return resolveUnrequestedOne.call(this, value);
+	}
+
+	async resolveRemoteUserObjects(value) {
+		const collection = await resolveCollection.call(this, value);
+
+		return collection.filter(element => !this.requesting.has(element)).map(element => {
+			if (typeof element === 'string') {
+				const object = RemoteUserObject.findOne({ uri: element });
+
+				if (object !== null) {
+					return object;
+				}
+			}
+
+			return resolveUnrequestedOne.call(this, element);
+		});
+	}
+}
diff --git a/src/common/remote/activitypub/type.ts b/src/common/remote/activitypub/type.ts
new file mode 100644
index 000000000..5c4750e14
--- /dev/null
+++ b/src/common/remote/activitypub/type.ts
@@ -0,0 +1,3 @@
+export type IObject = {
+	type: string;
+}
diff --git a/src/common/remote/resolve-user.ts b/src/common/remote/resolve-user.ts
new file mode 100644
index 000000000..13d155830
--- /dev/null
+++ b/src/common/remote/resolve-user.ts
@@ -0,0 +1,26 @@
+import { toUnicode, toASCII } from 'punycode';
+import User from '../../models/user';
+import resolvePerson from './activitypub/resolve-person';
+import webFinger from './webfinger';
+
+export default async (username, host, option) => {
+	const usernameLower = username.toLowerCase();
+	const hostLowerAscii = toASCII(host).toLowerCase();
+	const hostLower = toUnicode(hostLowerAscii);
+
+	let user = await User.findOne({ usernameLower, hostLower }, option);
+
+	if (user === null) {
+		const acctLower = `${usernameLower}@${hostLowerAscii}`;
+
+		const finger = await webFinger(acctLower, acctLower);
+		const self = finger.links.find(link => link.rel && link.rel.toLowerCase() === 'self');
+		if (!self) {
+			throw new Error;
+		}
+
+		user = await resolvePerson(self.href, usernameLower, hostLower, acctLower);
+	}
+
+	return user;
+};
diff --git a/src/common/remote/webfinger.ts b/src/common/remote/webfinger.ts
new file mode 100644
index 000000000..23f0aaa55
--- /dev/null
+++ b/src/common/remote/webfinger.ts
@@ -0,0 +1,25 @@
+const WebFinger = require('webfinger.js');
+
+const webFinger = new WebFinger({});
+
+type ILink = {
+  href: string;
+  rel: string;
+}
+
+type IWebFinger = {
+  links: Array<ILink>;
+  subject: string;
+}
+
+export default (query, verifier): Promise<IWebFinger> => new Promise((res, rej) => webFinger.lookup(query, (error, result) => {
+	if (error) {
+		return rej(error);
+	}
+
+	if (result.object.subject.toLowerCase().replace(/^acct:/, '') !== verifier) {
+		return rej('WebFinger verfification failed');
+	}
+
+	res(result.object);
+}));
diff --git a/src/models/remote-user-object.ts b/src/models/remote-user-object.ts
new file mode 100644
index 000000000..fb5b337c9
--- /dev/null
+++ b/src/models/remote-user-object.ts
@@ -0,0 +1,15 @@
+import * as mongodb from 'mongodb';
+import db from '../db/mongodb';
+
+const RemoteUserObject = db.get<IRemoteUserObject>('remoteUserObjects');
+
+export default RemoteUserObject;
+
+export type IRemoteUserObject = {
+	_id: mongodb.ObjectID;
+	uri: string;
+	object: {
+		$ref: string;
+		$id: mongodb.ObjectID;
+	}
+};
diff --git a/src/models/user.ts b/src/models/user.ts
index 4fbfdec90..4728682d6 100644
--- a/src/models/user.ts
+++ b/src/models/user.ts
@@ -97,6 +97,9 @@ export type IUser = {
 	account: ILocalAccount | IRemoteAccount;
 };
 
+export type ILocalUser = IUser & { account: ILocalAccount };
+export type IRemoteUser = IUser & { account: IRemoteAccount };
+
 export function init(user): IUser {
 	user._id = new mongo.ObjectID(user._id);
 	user.avatarId = new mongo.ObjectID(user.avatarId);
diff --git a/src/processor/http/index.ts b/src/processor/http/index.ts
new file mode 100644
index 000000000..da942ad2a
--- /dev/null
+++ b/src/processor/http/index.ts
@@ -0,0 +1,9 @@
+import performActivityPub from './perform-activitypub';
+import reportGitHubFailure from './report-github-failure';
+
+const handlers = {
+  performActivityPub,
+  reportGitHubFailure,
+};
+
+export default (job, done) => handlers[job.data.type](job).then(() => done(), done);
diff --git a/src/processor/http/perform-activitypub.ts b/src/processor/http/perform-activitypub.ts
new file mode 100644
index 000000000..5b1a02173
--- /dev/null
+++ b/src/processor/http/perform-activitypub.ts
@@ -0,0 +1,6 @@
+import User from '../../models/user';
+import act from '../../common/remote/activitypub/act';
+
+export default ({ data }, done) => User.findOne({ _id: data.actor })
+	.then(actor => act(actor, data.outbox))
+	.then(() => done(), done);
diff --git a/src/processor/report-github-failure.ts b/src/processor/http/report-github-failure.ts
similarity index 87%
rename from src/processor/report-github-failure.ts
rename to src/processor/http/report-github-failure.ts
index 610ffe276..53924a0fb 100644
--- a/src/processor/report-github-failure.ts
+++ b/src/processor/http/report-github-failure.ts
@@ -1,6 +1,6 @@
 import * as request from 'request';
-import User from '../models/user';
-const createPost = require('../server/api/endpoints/posts/create');
+import User from '../../models/user';
+const createPost = require('../../server/api/endpoints/posts/create');
 
 export default ({ data }, done) => {
 	const asyncBot = User.findOne({ _id: data.userId });
diff --git a/src/processor/index.ts b/src/processor/index.ts
index f06cf24e8..cd271d372 100644
--- a/src/processor/index.ts
+++ b/src/processor/index.ts
@@ -1,4 +1,13 @@
 import queue from '../queue';
-import reportGitHubFailure from './report-github-failure';
+import http from './http';
 
-export default () => queue.process('gitHubFailureReport', reportGitHubFailure);
+/*
+	256 is the default concurrency limit of Mozilla Firefox and Google
+	Chromium.
+
+	a8af215e691f3a2205a3758d2d96e9d328e100ff - chromium/src.git - Git at Google
+	https://chromium.googlesource.com/chromium/src.git/+/a8af215e691f3a2205a3758d2d96e9d328e100ff
+	Network.http.max-connections - MozillaZine Knowledge Base
+	http://kb.mozillazine.org/Network.http.max-connections
+*/
+export default () => queue.process('http', 256, http);
diff --git a/src/server/api/common/notify.ts b/src/server/api/common/notify.ts
index f90506cf3..69bf8480b 100644
--- a/src/server/api/common/notify.ts
+++ b/src/server/api/common/notify.ts
@@ -1,7 +1,7 @@
 import * as mongo from 'mongodb';
 import Notification from '../../../models/notification';
 import Mute from '../../../models/mute';
-import event from '../event';
+import event from '../../../common/event';
 import { pack } from '../../../models/notification';
 
 export default (
diff --git a/src/server/api/common/read-messaging-message.ts b/src/server/api/common/read-messaging-message.ts
index f728130bb..127ea1865 100644
--- a/src/server/api/common/read-messaging-message.ts
+++ b/src/server/api/common/read-messaging-message.ts
@@ -1,9 +1,9 @@
 import * as mongo from 'mongodb';
 import Message from '../../../models/messaging-message';
 import { IMessagingMessage as IMessage } from '../../../models/messaging-message';
-import publishUserStream from '../event';
-import { publishMessagingStream } from '../event';
-import { publishMessagingIndexStream } from '../event';
+import publishUserStream from '../../../common/event';
+import { publishMessagingStream } from '../../../common/event';
+import { publishMessagingIndexStream } from '../../../common/event';
 
 /**
  * Mark as read message(s)
diff --git a/src/server/api/common/read-notification.ts b/src/server/api/common/read-notification.ts
index 27632c7ec..9b2012182 100644
--- a/src/server/api/common/read-notification.ts
+++ b/src/server/api/common/read-notification.ts
@@ -1,6 +1,6 @@
 import * as mongo from 'mongodb';
 import { default as Notification, INotification } from '../../../models/notification';
-import publishUserStream from '../event';
+import publishUserStream from '../../../common/event';
 
 /**
  * Mark as read notification(s)
diff --git a/src/server/api/endpoints/drive/files/create.ts b/src/server/api/endpoints/drive/files/create.ts
index 53c8c7067..cf4e35cd1 100644
--- a/src/server/api/endpoints/drive/files/create.ts
+++ b/src/server/api/endpoints/drive/files/create.ts
@@ -3,7 +3,7 @@
  */
 import $ from 'cafy';
 import { validateFileName, pack } from '../../../../../models/drive-file';
-import create from '../../../common/drive/add-file';
+import create from '../../../../../common/drive/add-file';
 
 /**
  * Create a file
diff --git a/src/server/api/endpoints/drive/files/update.ts b/src/server/api/endpoints/drive/files/update.ts
index 836b4cfcd..5d0b915f9 100644
--- a/src/server/api/endpoints/drive/files/update.ts
+++ b/src/server/api/endpoints/drive/files/update.ts
@@ -4,7 +4,7 @@
 import $ from 'cafy';
 import DriveFolder from '../../../../../models/drive-folder';
 import DriveFile, { validateFileName, pack } from '../../../../../models/drive-file';
-import { publishDriveStream } from '../../../event';
+import { publishDriveStream } from '../../../../../common/event';
 
 /**
  * Update a file
diff --git a/src/server/api/endpoints/drive/files/upload_from_url.ts b/src/server/api/endpoints/drive/files/upload_from_url.ts
index 7262f09bb..01d875055 100644
--- a/src/server/api/endpoints/drive/files/upload_from_url.ts
+++ b/src/server/api/endpoints/drive/files/upload_from_url.ts
@@ -3,7 +3,7 @@
  */
 import $ from 'cafy';
 import { pack } from '../../../../../models/drive-file';
-import uploadFromUrl from '../../../common/drive/upload_from_url';
+import uploadFromUrl from '../../../../../common/drive/upload_from_url';
 
 /**
  * Create a file from a URL
diff --git a/src/server/api/endpoints/drive/folders/create.ts b/src/server/api/endpoints/drive/folders/create.ts
index 24e035930..bd3b0a0b1 100644
--- a/src/server/api/endpoints/drive/folders/create.ts
+++ b/src/server/api/endpoints/drive/folders/create.ts
@@ -3,7 +3,7 @@
  */
 import $ from 'cafy';
 import DriveFolder, { isValidFolderName, pack } from '../../../../../models/drive-folder';
-import { publishDriveStream } from '../../../event';
+import { publishDriveStream } from '../../../../../common/event';
 
 /**
  * Create drive folder
diff --git a/src/server/api/endpoints/drive/folders/update.ts b/src/server/api/endpoints/drive/folders/update.ts
index 6c5a5c376..5ac81e5b5 100644
--- a/src/server/api/endpoints/drive/folders/update.ts
+++ b/src/server/api/endpoints/drive/folders/update.ts
@@ -3,7 +3,7 @@
  */
 import $ from 'cafy';
 import DriveFolder, { isValidFolderName, pack } from '../../../../../models/drive-folder';
-import { publishDriveStream } from '../../../event';
+import { publishDriveStream } from '../../../../../common/event';
 
 /**
  * Update a folder
diff --git a/src/server/api/endpoints/following/create.ts b/src/server/api/endpoints/following/create.ts
index 1e24388a7..a689250e3 100644
--- a/src/server/api/endpoints/following/create.ts
+++ b/src/server/api/endpoints/following/create.ts
@@ -5,7 +5,7 @@ import $ from 'cafy';
 import User, { pack as packUser } from '../../../../models/user';
 import Following from '../../../../models/following';
 import notify from '../../common/notify';
-import event from '../../event';
+import event from '../../../../common/event';
 
 /**
  * Follow a user
diff --git a/src/server/api/endpoints/following/delete.ts b/src/server/api/endpoints/following/delete.ts
index 7fc5f477f..ecca27d57 100644
--- a/src/server/api/endpoints/following/delete.ts
+++ b/src/server/api/endpoints/following/delete.ts
@@ -4,7 +4,7 @@
 import $ from 'cafy';
 import User, { pack as packUser } from '../../../../models/user';
 import Following from '../../../../models/following';
-import event from '../../event';
+import event from '../../../../common/event';
 
 /**
  * Unfollow a user
diff --git a/src/server/api/endpoints/i/regenerate_token.ts b/src/server/api/endpoints/i/regenerate_token.ts
index c35778ac0..0af622fd9 100644
--- a/src/server/api/endpoints/i/regenerate_token.ts
+++ b/src/server/api/endpoints/i/regenerate_token.ts
@@ -4,7 +4,7 @@
 import $ from 'cafy';
 import * as bcrypt from 'bcryptjs';
 import User from '../../../../models/user';
-import event from '../../event';
+import event from '../../../../common/event';
 import generateUserToken from '../../common/generate-native-user-token';
 
 /**
diff --git a/src/server/api/endpoints/i/update.ts b/src/server/api/endpoints/i/update.ts
index 8e198f3ad..b465e763e 100644
--- a/src/server/api/endpoints/i/update.ts
+++ b/src/server/api/endpoints/i/update.ts
@@ -3,7 +3,7 @@
  */
 import $ from 'cafy';
 import User, { isValidName, isValidDescription, isValidLocation, isValidBirthday, pack } from '../../../../models/user';
-import event from '../../event';
+import event from '../../../../common/event';
 import config from '../../../../conf';
 
 /**
diff --git a/src/server/api/endpoints/i/update_client_setting.ts b/src/server/api/endpoints/i/update_client_setting.ts
index 03867b401..79789e664 100644
--- a/src/server/api/endpoints/i/update_client_setting.ts
+++ b/src/server/api/endpoints/i/update_client_setting.ts
@@ -3,7 +3,7 @@
  */
 import $ from 'cafy';
 import User, { pack } from '../../../../models/user';
-import event from '../../event';
+import event from '../../../../common/event';
 
 /**
  * Update myself
diff --git a/src/server/api/endpoints/i/update_home.ts b/src/server/api/endpoints/i/update_home.ts
index 713cf9fcc..437f51d6f 100644
--- a/src/server/api/endpoints/i/update_home.ts
+++ b/src/server/api/endpoints/i/update_home.ts
@@ -3,7 +3,7 @@
  */
 import $ from 'cafy';
 import User from '../../../../models/user';
-import event from '../../event';
+import event from '../../../../common/event';
 
 module.exports = async (params, user) => new Promise(async (res, rej) => {
 	// Get 'home' parameter
diff --git a/src/server/api/endpoints/i/update_mobile_home.ts b/src/server/api/endpoints/i/update_mobile_home.ts
index 6f28cebf9..783ca09d1 100644
--- a/src/server/api/endpoints/i/update_mobile_home.ts
+++ b/src/server/api/endpoints/i/update_mobile_home.ts
@@ -3,7 +3,7 @@
  */
 import $ from 'cafy';
 import User from '../../../../models/user';
-import event from '../../event';
+import event from '../../../../common/event';
 
 module.exports = async (params, user) => new Promise(async (res, rej) => {
 	// Get 'home' parameter
diff --git a/src/server/api/endpoints/messaging/messages/create.ts b/src/server/api/endpoints/messaging/messages/create.ts
index 3d3b204da..b2b6c971d 100644
--- a/src/server/api/endpoints/messaging/messages/create.ts
+++ b/src/server/api/endpoints/messaging/messages/create.ts
@@ -9,8 +9,8 @@ import User from '../../../../../models/user';
 import Mute from '../../../../../models/mute';
 import DriveFile from '../../../../../models/drive-file';
 import { pack } from '../../../../../models/messaging-message';
-import publishUserStream from '../../../event';
-import { publishMessagingStream, publishMessagingIndexStream, pushSw } from '../../../event';
+import publishUserStream from '../../../../../common/event';
+import { publishMessagingStream, publishMessagingIndexStream, pushSw } from '../../../../../common/event';
 import html from '../../../../../common/text/html';
 import parse from '../../../../../common/text/parse';
 import config from '../../../../../conf';
diff --git a/src/server/api/endpoints/notifications/mark_as_read_all.ts b/src/server/api/endpoints/notifications/mark_as_read_all.ts
index 3693ba87b..f9bc6ebf7 100644
--- a/src/server/api/endpoints/notifications/mark_as_read_all.ts
+++ b/src/server/api/endpoints/notifications/mark_as_read_all.ts
@@ -2,7 +2,7 @@
  * Module dependencies
  */
 import Notification from '../../../../models/notification';
-import event from '../../event';
+import event from '../../../../common/event';
 
 /**
  * Mark as read all notifications
diff --git a/src/server/api/endpoints/othello/match.ts b/src/server/api/endpoints/othello/match.ts
index 03168095d..992b93d41 100644
--- a/src/server/api/endpoints/othello/match.ts
+++ b/src/server/api/endpoints/othello/match.ts
@@ -2,7 +2,7 @@ import $ from 'cafy';
 import Matching, { pack as packMatching } from '../../../../models/othello-matching';
 import OthelloGame, { pack as packGame } from '../../../../models/othello-game';
 import User from '../../../../models/user';
-import publishUserStream, { publishOthelloStream } from '../../event';
+import publishUserStream, { publishOthelloStream } from '../../../../common/event';
 import { eighteight } from '../../../../common/othello/maps';
 
 module.exports = (params, user) => new Promise(async (res, rej) => {
diff --git a/src/server/api/endpoints/posts/create.ts b/src/server/api/endpoints/posts/create.ts
index 5342f7772..42901ebcb 100644
--- a/src/server/api/endpoints/posts/create.ts
+++ b/src/server/api/endpoints/posts/create.ts
@@ -16,7 +16,7 @@ import ChannelWatching from '../../../../models/channel-watching';
 import { pack } from '../../../../models/post';
 import notify from '../../common/notify';
 import watch from '../../common/watch-post';
-import event, { pushSw, publishChannelStream } from '../../event';
+import event, { pushSw, publishChannelStream } from '../../../../common/event';
 import getAcct from '../../../../common/user/get-acct';
 import parseAcct from '../../../../common/user/parse-acct';
 import config from '../../../../conf';
diff --git a/src/server/api/endpoints/posts/polls/vote.ts b/src/server/api/endpoints/posts/polls/vote.ts
index b970c05e8..98df074e5 100644
--- a/src/server/api/endpoints/posts/polls/vote.ts
+++ b/src/server/api/endpoints/posts/polls/vote.ts
@@ -7,7 +7,7 @@ import Post from '../../../../../models/post';
 import Watching from '../../../../../models/post-watching';
 import notify from '../../../common/notify';
 import watch from '../../../common/watch-post';
-import { publishPostStream } from '../../../event';
+import { publishPostStream } from '../../../../../common/event';
 
 /**
  * Vote poll of a post
diff --git a/src/server/api/endpoints/posts/reactions/create.ts b/src/server/api/endpoints/posts/reactions/create.ts
index 5d2b5a7ed..8db76d643 100644
--- a/src/server/api/endpoints/posts/reactions/create.ts
+++ b/src/server/api/endpoints/posts/reactions/create.ts
@@ -8,7 +8,7 @@ import { pack as packUser } from '../../../../../models/user';
 import Watching from '../../../../../models/post-watching';
 import notify from '../../../common/notify';
 import watch from '../../../common/watch-post';
-import { publishPostStream, pushSw } from '../../../event';
+import { publishPostStream, pushSw } from '../../../../../common/event';
 
 /**
  * React to a post
diff --git a/src/server/api/endpoints/users/show.ts b/src/server/api/endpoints/users/show.ts
index fd51d386b..9cd8716fe 100644
--- a/src/server/api/endpoints/users/show.ts
+++ b/src/server/api/endpoints/users/show.ts
@@ -2,49 +2,10 @@
  * Module dependencies
  */
 import $ from 'cafy';
-import { JSDOM } from 'jsdom';
-import { toUnicode, toASCII } from 'punycode';
-import uploadFromUrl from '../../common/drive/upload_from_url';
-import User, { pack, validateUsername, isValidName, isValidDescription } from '../../../../models/user';
-const request = require('request-promise-native');
-const WebFinger = require('webfinger.js');
+import User, { pack } from '../../../../models/user';
+import resolveRemoteUser from '../../../../common/remote/resolve-user';
 
-const webFinger = new WebFinger({});
-
-async function getCollectionCount(url) {
-	if (!url) {
-		return null;
-	}
-
-	try {
-		const collection = await request({ url, json: true });
-		return collection ? collection.totalItems : null;
-	} catch (exception) {
-		return null;
-	}
-}
-
-function findUser(q) {
-	return User.findOne(q, {
-		fields: {
-			data: false
-		}
-	});
-}
-
-function webFingerAndVerify(query, verifier) {
-	return new Promise((res, rej) => webFinger.lookup(query, (error, result) => {
-		if (error) {
-			return rej(error);
-		}
-
-		if (result.object.subject.toLowerCase().replace(/^acct:/, '') !== verifier) {
-			return rej('WebFinger verfification failed');
-		}
-
-		res(result.object);
-	}));
-}
+const cursorOption = { fields: { data: false } };
 
 /**
  * Show a user
@@ -74,124 +35,17 @@ module.exports = (params, me) => new Promise(async (res, rej) => {
 
 	// Lookup user
 	if (typeof host === 'string') {
-		const usernameLower = username.toLowerCase();
-		const hostLowerAscii = toASCII(host).toLowerCase();
-		const hostLower = toUnicode(hostLowerAscii);
-
-		user = await findUser({ usernameLower, hostLower });
-
-		if (user === null) {
-			const acctLower = `${usernameLower}@${hostLowerAscii}`;
-			let activityStreams;
-			let finger;
-			let followersCount;
-			let followingCount;
-			let postsCount;
-
-			if (!validateUsername(username)) {
-				return rej('username validation failed');
-			}
-
-			try {
-				finger = await webFingerAndVerify(acctLower, acctLower);
-			} catch (exception) {
-				return rej('WebFinger lookup failed');
-			}
-
-			const self = finger.links.find(link => link.rel && link.rel.toLowerCase() === 'self');
-			if (!self) {
-				return rej('WebFinger has no reference to self representation');
-			}
-
-			try {
-				activityStreams = await request({
-					url: self.href,
-					headers: {
-						Accept: 'application/activity+json, application/ld+json'
-					},
-					json: true
-				});
-			} catch (exception) {
-				return rej('failed to retrieve ActivityStreams representation');
-			}
-
-			if (!(activityStreams &&
-				(Array.isArray(activityStreams['@context']) ?
-					activityStreams['@context'].includes('https://www.w3.org/ns/activitystreams') :
-					activityStreams['@context'] === 'https://www.w3.org/ns/activitystreams') &&
-				activityStreams.type === 'Person' &&
-				typeof activityStreams.preferredUsername === 'string' &&
-				activityStreams.preferredUsername.toLowerCase() === usernameLower &&
-				isValidName(activityStreams.name) &&
-				isValidDescription(activityStreams.summary)
-			)) {
-				return rej('failed ActivityStreams validation');
-			}
-
-			try {
-				[followersCount, followingCount, postsCount] = await Promise.all([
-					getCollectionCount(activityStreams.followers),
-					getCollectionCount(activityStreams.following),
-					getCollectionCount(activityStreams.outbox),
-					webFingerAndVerify(activityStreams.id, acctLower),
-				]);
-			} catch (exception) {
-				return rej('failed to fetch assets');
-			}
-
-			const summaryDOM = JSDOM.fragment(activityStreams.summary);
-
-			// Create user
-			user = await User.insert({
-				avatarId: null,
-				bannerId: null,
-				createdAt: new Date(),
-				description: summaryDOM.textContent,
-				followersCount,
-				followingCount,
-				name: activityStreams.name,
-				postsCount,
-				driveCapacity: 1024 * 1024 * 8, // 8MiB
-				username,
-				usernameLower,
-				host: toUnicode(finger.subject.replace(/^.*?@/, '')),
-				hostLower,
-				account: {
-					uri: activityStreams.id,
-				},
-			});
-
-			const [icon, image] = await Promise.all([
-				activityStreams.icon,
-				activityStreams.image,
-			].map(async image => {
-				if (!image || image.type !== 'Image') {
-					return { _id: null };
-				}
-
-				try {
-					return await uploadFromUrl(image.url, user);
-				} catch (exception) {
-					return { _id: null };
-				}
-			}));
-
-			User.update({ _id: user._id }, {
-				$set: {
-					avatarId: icon._id,
-					bannerId: image._id,
-				},
-			});
-
-			user.avatarId = icon._id;
-			user.bannerId = icon._id;
+		try {
+			user = await resolveRemoteUser(username, host, cursorOption);
+		} catch (exception) {
+			return rej('failed to resolve remote user');
 		}
 	} else {
 		const q = userId !== undefined
 			? { _id: userId }
 			: { usernameLower: username.toLowerCase(), host: null };
 
-		user = await findUser(q);
+		user = await User.findOne(q, cursorOption);
 
 		if (user === null) {
 			return rej('user not found');
diff --git a/src/server/api/private/signin.ts b/src/server/api/private/signin.ts
index d78fa11b8..4b7064491 100644
--- a/src/server/api/private/signin.ts
+++ b/src/server/api/private/signin.ts
@@ -3,7 +3,7 @@ import * as bcrypt from 'bcryptjs';
 import * as speakeasy from 'speakeasy';
 import { default as User, ILocalAccount, IUser } from '../../../models/user';
 import Signin, { pack } from '../../../models/signin';
-import event from '../event';
+import event from '../../../common/event';
 import signin from '../common/signin';
 import config from '../../../conf';
 
diff --git a/src/server/api/service/github.ts b/src/server/api/service/github.ts
index a2359cfb6..b4068c729 100644
--- a/src/server/api/service/github.ts
+++ b/src/server/api/service/github.ts
@@ -42,7 +42,8 @@ module.exports = async (app: express.Application) => {
 				const commit = event.commit;
 				const parent = commit.parents[0];
 
-				queue.create('gitHubFailureReport', {
+				queue.create('http', {
+					type: 'gitHubFailureReport',
 					userId: bot._id,
 					parentUrl: parent.url,
 					htmlUrl: commit.html_url,
diff --git a/src/server/api/service/twitter.ts b/src/server/api/service/twitter.ts
index d77341db2..73822b0bd 100644
--- a/src/server/api/service/twitter.ts
+++ b/src/server/api/service/twitter.ts
@@ -6,7 +6,7 @@ import * as uuid from 'uuid';
 import autwh from 'autwh';
 import redis from '../../../db/redis';
 import User, { pack } from '../../../models/user';
-import event from '../event';
+import event from '../../../common/event';
 import config from '../../../conf';
 import signin from '../common/signin';
 
diff --git a/src/server/api/stream/othello-game.ts b/src/server/api/stream/othello-game.ts
index b6a251c4c..b11915f8f 100644
--- a/src/server/api/stream/othello-game.ts
+++ b/src/server/api/stream/othello-game.ts
@@ -2,7 +2,7 @@ import * as websocket from 'websocket';
 import * as redis from 'redis';
 import * as CRC32 from 'crc-32';
 import OthelloGame, { pack } from '../../../models/othello-game';
-import { publishOthelloGameStream } from '../event';
+import { publishOthelloGameStream } from '../../../common/event';
 import Othello from '../../../common/othello/core';
 import * as maps from '../../../common/othello/maps';
 import { ParsedUrlQuery } from 'querystring';
diff --git a/src/server/api/stream/othello.ts b/src/server/api/stream/othello.ts
index 4205afae7..1cf9a1494 100644
--- a/src/server/api/stream/othello.ts
+++ b/src/server/api/stream/othello.ts
@@ -2,7 +2,7 @@ import * as mongo from 'mongodb';
 import * as websocket from 'websocket';
 import * as redis from 'redis';
 import Matching, { pack } from '../../../models/othello-matching';
-import publishUserStream from '../event';
+import publishUserStream from '../../../common/event';
 
 export default function(request: websocket.request, connection: websocket.connection, subscriber: redis.RedisClient, user: any): void {
 	// Subscribe othello stream