[mastodon-client] GET /v2/search

packages/backend/src/server/api/mastodon/converters.ts

@@ -50,6 +50,13 @@ export function convertRelationship(relationship: Entity.Relationship) {
 	return simpleConvert(relationship);
 }
 
+export function convertSearch(search: MastodonEntity.Search) {
+	search.accounts = search.accounts.map(p => convertAccount(p));
+	search.statuses = search.statuses.map(p => convertStatus(p));
+	return search;
+}
+
+
 export function convertStatus(status: MastodonEntity.Status) {
 	status.account = convertAccount(status.account);
 	status.id = convertId(status.id, IdType.MastodonId);

packages/backend/src/server/api/mastodon/endpoints/search.ts

@@ -3,8 +3,11 @@ import Router from "@koa/router";
 import { getClient } from "../index.js";
 import axios from "axios";
 import { Converter } from "megalodon";
-import { convertPaginationArgsIds, limitToInt } from "./timeline.js";
+import { argsToBools, convertPaginationArgsIds, limitToInt, normalizeUrlQuery } from "./timeline.js";
-import { convertAccount, convertStatus } from "../converters.js";
+import { convertAccount, convertSearch, convertStatus } from "../converters.js";
+import authenticate from "@/server/api/authenticate.js";
+import { UserHelpers } from "@/server/api/mastodon/helpers/user.js";
+import { SearchHelpers } from "@/server/api/mastodon/helpers/search.js";
 
 export function setupEndpointsSearch(router: Router): void {
 	router.get("/v1/search", async (ctx) => {
@@ -24,36 +27,24 @@ export function setupEndpointsSearch(router: Router): void {
 		}
 	});
 	router.get("/v2/search", async (ctx) => {
-		const BASE_URL = `${ctx.request.protocol}://${ctx.request.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
 		try {
-			const query: any = convertPaginationArgsIds(limitToInt(ctx.query));
+			const auth = await authenticate(ctx.headers.authorization, null);
-			const type = query.type;
+			const user = auth[0] ?? undefined;
-			const acct =
-				!type || type === "accounts"
-					? await client.search(query.q, "accounts", query)
-					: null;
-			const stat =
-				!type || type === "statuses"
-					? await client.search(query.q, "statuses", query)
-					: null;
-			const tags =
-				!type || type === "hashtags"
-					? await client.search(query.q, "hashtags", query)
-					: null;
 
-			ctx.body = {
+			if (!user) {
-				accounts:
+				ctx.status = 401;
-					acct?.data?.accounts.map((account) => convertAccount(account)) ?? [],
+				return;
-				statuses:
+			}
-					stat?.data?.statuses.map((status) => convertStatus(status)) ?? [],
+
-				hashtags: tags?.data?.hashtags ?? [],
+			const args = normalizeUrlQuery(convertPaginationArgsIds(argsToBools(limitToInt(ctx.query), ['resolve', 'following', 'exclude_unreviewed'])));
-			};
+			const cache = UserHelpers.getFreshAccountCache();
+			const result = await SearchHelpers.search(user, args.q, args.type, args.resolve, args.following, args.account_id, args['exclude_unreviewed'], args.max_id, args.min_id, args.limit, args.offset, cache);
+
+			ctx.body = convertSearch(result);
 		} catch (e: any) {
 			console.error(e);
-			ctx.status = 401;
+			ctx.status = 400;
-			ctx.body = e.response.data;
+			ctx.body = { error: e.message };
 		}
 	});
 	router.get("/v1/trends/statuses", async (ctx) => {

packages/backend/src/server/api/mastodon/entities/results.ts

@@ -3,7 +3,7 @@
 /// <reference path="tag.ts" />
 
 namespace MastodonEntity {
-	export type Results = {
+	export type Search = {
 		accounts: Array<Account>;
 		statuses: Array<Status>;
 		hashtags: Array<Tag>;

packages/backend/src/server/api/mastodon/helpers/search.ts

@@ -0,0 +1,410 @@
+import es from "@/db/elasticsearch.js";
+import sonic from "@/db/sonic.js";
+import meilisearch, { MeilisearchNote } from "@/db/meilisearch.js";
+import { Followings, Hashtags, Notes, Users } from "@/models/index.js";
+import { sqlLikeEscape } from "@/misc/sql-like-escape.js";
+import { generateVisibilityQuery } from "@/server/api/common/generate-visibility-query.js";
+import { generateMutedUserQuery } from "@/server/api/common/generate-muted-user-query.js";
+import { generateBlockedUserQuery } from "@/server/api/common/generate-block-query.js";
+import { Note } from "@/models/entities/note.js";
+import { PaginationHelpers } from "@/server/api/mastodon/helpers/pagination.js";
+import { ILocalUser, User } from "@/models/entities/user.js";
+import { Brackets, In, IsNull } from "typeorm";
+import { awaitAll } from "@/prelude/await-all.js";
+import { AccountCache, UserHelpers } from "@/server/api/mastodon/helpers/user.js";
+import { NoteConverter } from "@/server/api/mastodon/converters/note.js";
+import Resolver from "@/remote/activitypub/resolver.js";
+import { getApId, isActor, isPost } from "@/remote/activitypub/type.js";
+import DbResolver from "@/remote/activitypub/db-resolver.js";
+import { createPerson } from "@/remote/activitypub/models/person.js";
+import { UserConverter } from "@/server/api/mastodon/converters/user.js";
+import { resolveUser } from "@/remote/resolve-user.js";
+import { createNote } from "@/remote/activitypub/models/note.js";
+import { getUser } from "@/server/api/common/getters.js";
+import config from "@/config/index.js";
+import { Hashtag } from "@/models/entities/hashtag.js";
+
+export class SearchHelpers {
+	public static async search(user: ILocalUser, q: string | undefined, type: string | undefined, resolve: boolean = false, following: boolean = false, accountId: string | undefined, excludeUnreviewed: boolean = false, maxId: string | undefined, minId: string | undefined, limit: number = 20, offset: number | undefined, cache: AccountCache = UserHelpers.getFreshAccountCache()): Promise<MastodonEntity.Search> {
+		if (q === undefined || q.trim().length === 0) throw new Error('Search query cannot be empty');
+		if (limit > 40) limit = 40;
+		const notes = type === 'statuses' || !type ? this.searchNotes(user, q, resolve, following, accountId, maxId, minId, limit, offset) : [];
+		const users = type === 'accounts' || !type ? this.searchUsers(user, q, resolve, following, maxId, minId, limit, offset) : [];
+		const tags = type === 'hashtags' || !type ? this.searchTags(q, excludeUnreviewed, limit, offset) : [];
+
+		const result = {
+			statuses: Promise.resolve(notes).then(p => NoteConverter.encodeMany(p, user, cache)),
+			accounts: Promise.resolve(users).then(p => UserConverter.encodeMany(p, cache)),
+			hashtags: Promise.resolve(tags)
+		};
+
+		return awaitAll(result);
+	}
+
+	private static async searchUsers(user: ILocalUser, q: string, resolve: boolean, following: boolean, maxId: string | undefined, minId: string | undefined, limit: number, offset: number | undefined): Promise<User[]> {
+		if (resolve) {
+			try {
+				if (q.startsWith('https://') || q.startsWith('http://')) {
+					// try resolving locally first
+					const dbResolver = new DbResolver();
+					const dbResult = await dbResolver.getUserFromApId(q);
+					if (dbResult) return [dbResult];
+
+					// ask remote
+					const resolver = new Resolver();
+					resolver.setUser(user);
+					const object = await resolver.resolve(q);
+					if (q !== object.id) {
+						const result = await dbResolver.getUserFromApId(getApId(object));
+						if (result) return [result];
+					}
+					return isActor(object) ? Promise.all([createPerson(getApId(object), resolver.reset())]) : [];
+				}
+				else {
+					let match = q.match(/^@?(?<user>[a-zA-Z0-9_]+)@(?<host>[a-zA-Z0-9-.]+\.[a-zA-Z0-9-]+)$/);
+					if (!match) match = q.match(/^@(?<user>[a-zA-Z0-9_]+)$/)
+					if (match) {
+						// check if user is already in database
+						const dbResult = await Users.findBy({usernameLower: match.groups!.user.toLowerCase(), host: match.groups?.host ?? IsNull()});
+						if (dbResult) return dbResult;
+
+						const result = await resolveUser(match.groups!.user.toLowerCase(), match.groups?.host ?? null);
+						if (result) return [result];
+
+						// no matches found
+						return [];
+					}
+				}
+			}
+			catch (e: any) {
+				console.log(`[mastodon-client] resolve user '${q}' failed: ${e.message}`);
+				return [];
+			}
+		}
+
+		const query = PaginationHelpers.makePaginationQuery(
+			Users.createQueryBuilder("user"),
+			undefined,
+			minId,
+			maxId,
+		);
+
+		if (following) {
+			const followingQuery = Followings.createQueryBuilder("following")
+				.select("following.followeeId")
+				.where("following.followerId = :followerId", {followerId: user.id});
+
+			query.andWhere(
+				new Brackets((qb) => {
+					qb.where(`user.id IN (${followingQuery.getQuery()} UNION ALL VALUES (:meId))`, {meId: user.id});
+				}),
+			);
+		}
+
+		query.andWhere(
+			new Brackets((qb) => {
+				qb.where("user.name ILIKE :q", { q: `%${sqlLikeEscape(q)}%` });
+				qb.orWhere("user.usernameLower ILIKE :q", { q: `%${sqlLikeEscape(q)}%` });
+			})
+		);
+
+		query.orderBy({'user.notesCount': 'DESC'});
+
+		return query.skip(offset ?? 0).take(limit).getMany().then(p => minId ? p.reverse() : p);
+	}
+
+	private static async searchNotes(user: ILocalUser, q: string, resolve: boolean, following: boolean, accountId: string | undefined, maxId: string | undefined, minId: string | undefined, limit: number, offset: number | undefined): Promise<Note[]> {
+		if (accountId && following) throw new Error("The 'following' and 'accountId' parameters cannot be used simultaneously");
+
+		if (resolve) {
+			try {
+				if (q.startsWith('https://') || q.startsWith('http://')) {
+					// try resolving locally first
+					const dbResolver = new DbResolver();
+					const dbResult = await dbResolver.getNoteFromApId(q);
+					if (dbResult) return [dbResult];
+
+					// ask remote
+					const resolver = new Resolver();
+					resolver.setUser(user);
+					const object = await resolver.resolve(q);
+					if (q !== object.id) {
+						const result = await dbResolver.getNoteFromApId(getApId(object));
+						if (result) return [result];
+					}
+
+					return isPost(object) ? createNote(getApId(object), resolver.reset(), true).then(p => p ? [p] : []) : [];
+				}
+			}
+			catch (e: any) {
+				console.log(`[mastodon-client] resolve note '${q}' failed: ${e.message}`);
+				return [];
+			}
+		}
+
+		// Try sonic search first, unless we have advanced filters
+		if (sonic && !accountId && !following) {
+			let start = offset ?? 0;
+			const chunkSize = 100;
+
+			// Use sonic to fetch and step through all search results that could match the requirements
+			const ids = [];
+			while (true) {
+				const results = await sonic.search.query(
+					sonic.collection,
+					sonic.bucket,
+					q,
+					{
+						limit: chunkSize,
+						offset: start,
+					},
+				);
+
+				start += chunkSize;
+
+				if (results.length === 0) {
+					break;
+				}
+
+				const res = results
+					.map((k) => JSON.parse(k))
+					.filter((key) => {
+						if (minId && key.id < minId) return false;
+						if (maxId && key.id > maxId) return false;
+						return true;
+					})
+					.map((key) => key.id);
+
+				ids.push(...res);
+			}
+
+			// Sort all the results by note id DESC (newest first)
+			ids.sort((a, b) => b - a);
+
+			// Fetch the notes from the database until we have enough to satisfy the limit
+			start = 0;
+			const found = [];
+			while (found.length < limit && start < ids.length) {
+				const chunk = ids.slice(start, start + chunkSize);
+
+				const query = Notes.createQueryBuilder("note")
+					.where({id: In(chunk)})
+					.orderBy({id: "DESC"})
+
+				generateVisibilityQuery(query, user);
+
+				if (!accountId) {
+					generateMutedUserQuery(query, user);
+					generateBlockedUserQuery(query, user);
+				}
+
+				if (following) {
+					const followingQuery = Followings.createQueryBuilder("following")
+						.select("following.followeeId")
+						.where("following.followerId = :followerId", {followerId: user.id});
+
+					query.andWhere(
+						new Brackets((qb) => {
+							qb.where(`note.userId IN (${followingQuery.getQuery()} UNION ALL VALUES (:meId))`, {meId: user.id});
+						}),
+					)
+				}
+
+				const notes: Note[] = await query.getMany();
+
+				found.push(...notes);
+				start += chunkSize;
+			}
+
+			// If we have more results than the limit, trim them
+			if (found.length > limit) {
+				found.length = limit;
+			}
+
+			return found;
+		}
+		// Try meilisearch next
+		else if (meilisearch) {
+			let start = 0;
+			const chunkSize = 100;
+
+			// Use meilisearch to fetch and step through all search results that could match the requirements
+			const ids = [];
+			if (accountId) {
+				const acc = await getUser(accountId);
+				const append = acc.host !== null ? `from:${acc.usernameLower}@${acc.host} ` : `from:${acc.usernameLower}`;
+				q = append + q;
+			}
+			if (following) {
+				q = `filter:following ${q}`;
+			}
+			while (true) {
+				const results = await meilisearch.search(q, chunkSize, start, user);
+
+				start += chunkSize;
+
+				if (results.hits.length === 0) {
+					break;
+				}
+
+				//TODO test this, it's the same logic the mk api uses but it seems, we need to make .hits already be a MeilisearchNote[] instead of forcing type checks to pass
+				const res = (results.hits as MeilisearchNote[])
+					.filter((key: MeilisearchNote) => {
+						if (accountId && key.userId !== accountId) return false;
+						if (minId && key.id < minId) return false;
+						if (maxId && key.id > maxId) return false;
+						return true;
+					})
+					.map((key) => key.id);
+
+				ids.push(...res);
+			}
+
+			// Sort all the results by note id DESC (newest first)
+			//FIXME: fix this sort function (is it even necessary?)
+			//ids.sort((a, b) => b - a);
+
+			// Fetch the notes from the database until we have enough to satisfy the limit
+			start = 0;
+			const found = [];
+			while (found.length < limit && start < ids.length) {
+				const chunk = ids.slice(start, start + chunkSize);
+
+				const query = Notes.createQueryBuilder("note")
+					.where({id: In(chunk)})
+					.orderBy({id: "DESC"})
+
+				generateVisibilityQuery(query, user);
+
+				if (!accountId) {
+					generateMutedUserQuery(query, user);
+					generateBlockedUserQuery(query, user);
+				}
+
+				const notes: Note[] = await query.getMany();
+
+				found.push(...notes);
+				start += chunkSize;
+			}
+
+			// If we have more results than the limit, trim them
+			if (found.length > limit) {
+				found.length = limit;
+			}
+
+			return found;
+		}
+		else if (es) {
+			const userQuery =
+				accountId != null
+					? [
+						{
+							term: {
+								userId: accountId,
+							},
+						},
+					]
+					: [];
+
+			const result = await es.search({
+				index: config.elasticsearch.index || "misskey_note",
+				body: {
+					size: limit,
+					from: offset,
+					query: {
+						bool: {
+							must: [
+								{
+									simple_query_string: {
+										fields: ["text"],
+										query: q.toLowerCase(),
+										default_operator: "and",
+									},
+								},
+								...userQuery,
+							],
+						},
+					},
+					sort: [
+						{
+							_doc: "desc",
+						},
+					],
+				},
+			});
+
+			const hits = result.body.hits.hits.map((hit: any) => hit._id);
+
+			if (hits.length === 0) return [];
+
+			// Fetch found notes
+			const notes = await Notes.find({
+				where: {
+					id: In(hits),
+				},
+				order: {
+					id: -1,
+				},
+			});
+
+			//TODO: test this
+			//FIXME: implement pagination
+			return notes;
+		}
+
+		// Fallback to database query
+		const query = PaginationHelpers.makePaginationQuery(
+			Notes.createQueryBuilder("note"),
+			undefined,
+			minId,
+			maxId,
+		);
+
+		if (accountId) {
+			query.andWhere("note.userId = :userId", { userId: accountId });
+		}
+
+		if (following) {
+			const followingQuery = Followings.createQueryBuilder("following")
+				.select("following.followeeId")
+				.where("following.followerId = :followerId", {followerId: user.id});
+
+			query.andWhere(
+				new Brackets((qb) => {
+					qb.where(`note.userId IN (${followingQuery.getQuery()} UNION ALL VALUES (:meId))`, {meId: user.id});
+				}),
+			)
+		}
+
+		query
+			.andWhere("note.text ILIKE :q", { q: `%${sqlLikeEscape(q)}%` })
+			.leftJoinAndSelect("note.renote", "renote");
+
+
+		generateVisibilityQuery(query, user);
+
+		if (!accountId) {
+			generateMutedUserQuery(query, user);
+			generateBlockedUserQuery(query, user);
+		}
+
+		return query.skip(offset ?? 0).take(limit).getMany().then(p => minId ? p.reverse() : p);
+	}
+
+	private static async searchTags(q: string, excludeUnreviewed: boolean, limit: number, offset: number | undefined): Promise<MastodonEntity.Tag[]> {
+		const tags = Hashtags.createQueryBuilder('tag')
+			.select('tag.name')
+			.distinctOn(['tag.name'])
+			.where("tag.name ILIKE :q", { q: `%${sqlLikeEscape(q)}%` })
+			.orderBy({'tag.name': 'ASC'})
+			.skip(offset ?? 0).take(limit).getMany();
+
+		return tags.then(p => p.map(tag => {
+			return {
+				name: tag.name,
+				url: `${config.url}/tags/${tag.name}`,
+				history: null
+			};
+		}));
+	}
+}