[mastodon-client] Improve html cache performance

packages/backend/src/remote/activitypub/models/person.ts

@@ -617,9 +617,10 @@ export async function updatePerson(
 		);
 	}
 
-	await UserProfiles.update(
+	// Get old profile to see if we need to update any matching html cache entries
-		{ userId: user.id },
+	const oldProfile = await UserProfiles.findOneBy({ userId: user.id });
-		{
+
+	const newProfile = {
 		url: url,
 		fields,
 		description: person._misskey_summary
@@ -628,9 +629,10 @@ export async function updatePerson(
 				? await htmlToMfm(truncate(person.summary, summaryLength), person.tag)
 				: null,
 		birthday: bday ? bday[0] : null,
-			location: person["vcard:Address"] || null,
+		location: person["vcard:Address"] || null
-		},
+	} as Partial<UserProfile>;
-	);
+
+	await UserProfiles.update({ userId: user.id }, newProfile);
 
 	publishInternalEvent("remoteUserUpdated", { id: user.id });
 
@@ -639,7 +641,7 @@ export async function updatePerson(
 
 	// Mentions update, then prewarm html cache
 	UserProfiles.updateMentions(user!.id)
-		.then(_ => UserConverter.prewarmCacheById(user!.id));
+		.then(_ => UserConverter.prewarmCacheById(user!.id, oldProfile));
 
 	// If the user in question is a follower, followers will also be updated.
 	await Followings.update(

packages/backend/src/server/api/mastodon/converters/note.ts

@@ -182,7 +182,7 @@ export class NoteConverter {
         return Promise.all(encoded);
     }
 
-    private static async aggregateData(notes: Note[], ctx: MastoContext): Promise<void> {
+    public static async aggregateData(notes: Note[], ctx: MastoContext): Promise<void> {
         if (notes.length === 0) return;
 
         const user = ctx.user as ILocalUser | null;
@@ -307,7 +307,7 @@ export class NoteConverter {
 
         return Promise.resolve(dbHit)
             .then(res => {
-                if (res === null || (res.updatedAt !== note.updatedAt)) {
+                if (res === null || (res.updatedAt?.getTime() !== note.updatedAt?.getTime())) {
                     this.prewarmCache(note);
                     return null;
                 }

packages/backend/src/server/api/mastodon/converters/notification.ts

@@ -1,4 +1,4 @@
-import { ILocalUser } from "@/models/entities/user.js";
+import { ILocalUser, User } from "@/models/entities/user.js";
 import { Notification } from "@/models/entities/notification.js";
 import { notificationTypes } from "@/types.js";
 import { UserConverter } from "@/server/api/mastodon/converters/user.js";
@@ -9,6 +9,8 @@ import { getNote } from "@/server/api/common/getters.js";
 import { getStubMastoContext, MastoContext } from "@/server/api/mastodon/index.js";
 import { Notifications } from "@/models/index.js";
 import isQuote from "@/misc/is-quote.js";
+import { unique } from "@/prelude/array.js";
+import { Note } from "@/models/entities/note.js";
 
 type NotificationType = typeof notificationTypes[number];
 
@@ -51,11 +53,21 @@ export class NotificationConverter {
     }
 
     public static async encodeMany(notifications: Notification[], ctx: MastoContext): Promise<MastodonEntity.Notification[]> {
+		await this.aggregateData(notifications, ctx);
         const encoded = notifications.map(u => this.encode(u, ctx));
         return Promise.all(encoded)
             .then(p => p.filter(n => n !== null) as MastodonEntity.Notification[]);
     }
 
+	private static async aggregateData(notifications: Notification[], ctx: MastoContext): Promise<void> {
+		if (notifications.length === 0) return;
+		const notes = unique(notifications.filter(p => p.note != null).map((n) => n.note as Note));
+		const users = unique(notifications.filter(p => p.notifier != null).map(n => n.notifier as User)
+			.concat(notifications.filter(p => p.notifiee != null).map(n => n.notifiee as User)));
+		await NoteConverter.aggregateData(notes, ctx);
+		await UserConverter.aggregateData(users, ctx);
+	}
+
     private static encodeNotificationType(t: NotificationType): MastodonEntity.NotificationType {
         //FIXME: Implement custom notification for followRequestAccepted
         //FIXME: Implement mastodon notification type 'update' on misskey side

packages/backend/src/server/api/mastodon/converters/user.ts

@@ -35,7 +35,7 @@ export class UserConverter {
             const cacheHit = cache.accounts.find(p => p.id == u.id);
             if (cacheHit) return cacheHit;
 
-            const identifier = `${u.id}:${(u.updatedAt ?? u.createdAt).getTime()}`;
+            const identifier = `${u.id}:${(u.lastFetchedAt ?? u.createdAt).getTime()}`;
             let fqn = `${u.username}@${u.host ?? config.domain}`;
             let acct = u.username;
             let acctUrl = `https://${u.host || config.host}/@${u.username}`;
@@ -243,7 +243,7 @@ export class UserConverter {
 
         return Promise.resolve(dbHit)
             .then(res => {
-                if (res === null || (res.updatedAt !== user.updatedAt ?? user.createdAt)) {
+                if (res === null || (res.updatedAt.getTime() !== (user.lastFetchedAt ?? user.createdAt).getTime())) {
                     this.prewarmCache(user, profile);
                     return null;
                 }
@@ -251,10 +251,21 @@ export class UserConverter {
             });
     }
 
-    public static async prewarmCache(user: User, profile?: UserProfile | null): Promise<void> {
+    public static async prewarmCache(user: User, profile?: UserProfile | null, oldProfile?: UserProfile | null): Promise<void> {
-        if (!config.htmlCache?.prewarm) return;
+        const identifier = `${user.id}:${(user.lastFetchedAt ?? user.createdAt).getTime()}`;
-        const identifier = `${user.id}:${(user.updatedAt ?? user.createdAt).getTime()}`;
         if (profile !== null) {
+			if (config.htmlCache?.dbFallback) {
+				if (profile === undefined) {
+					profile = await UserProfiles.findOneBy({ userId: user.id });
+				}
+				if (oldProfile !== undefined && profile?.fields === oldProfile?.fields && profile?.description === oldProfile?.description) {
+					HtmlUserCacheEntries.update({ userId: user.id }, { updatedAt: user.lastFetchedAt ?? user.createdAt });
+					return;
+				}
+			}
+
+			if (!config.htmlCache?.prewarm) return;
+
 			if (profile === undefined) {
 				profile = await UserProfiles.findOneBy({ userId: user.id });
 			}
@@ -267,7 +278,7 @@ export class UserConverter {
                 this.userBioHtmlCache.set(identifier, await bio);
 
                 if (config.htmlCache?.dbFallback)
-                    HtmlUserCacheEntries.upsert({ userId: user.id, bio: await bio }, ["userId"]);
+                    HtmlUserCacheEntries.upsert({ userId: user.id, updatedAt: user.lastFetchedAt ?? user.createdAt, bio: await bio }, ["userId"]);
             }
 
             if (await this.userFieldsHtmlCache.get(identifier) === undefined) {
@@ -275,12 +286,12 @@ export class UserConverter {
                 this.userFieldsHtmlCache.set(identifier, fields);
 
                 if (config.htmlCache?.dbFallback)
-                    HtmlUserCacheEntries.upsert({ userId: user.id, updatedAt: user.updatedAt ?? user.createdAt, fields: fields }, ["userId"]);
+                    HtmlUserCacheEntries.upsert({ userId: user.id, updatedAt: user.lastFetchedAt ?? user.createdAt, fields: fields }, ["userId"]);
             }
         }
     }
 
-    public static async prewarmCacheById(userId: string): Promise<void> {
+    public static async prewarmCacheById(userId: string, oldProfile?: UserProfile | null): Promise<void> {
-        await this.prewarmCache(await getUser(userId));
+        await this.prewarmCache(await getUser(userId), undefined, oldProfile);
     }
 }

packages/backend/src/server/api/mastodon/helpers/notification.ts

@@ -31,7 +31,10 @@ export class NotificationHelpers {
         if (accountId !== undefined)
             query.andWhere("notification.notifierId = :notifierId", { notifierId: accountId });
 
-        query.leftJoinAndSelect("notification.note", "note");
+        query
+			.leftJoinAndSelect("notification.note", "note")
+			.leftJoinAndSelect("notification.notifier", "notifier")
+			.leftJoinAndSelect("notification.notifiee", "notifiee");
 
         return PaginationHelpers.execQueryLinkPagination(query, limit, minId !== undefined, ctx);
     }