Lint + formatting

This commit is contained in:
PrivateGER 2023-05-26 03:06:41 +02:00 committed by PrivateGER
parent a8a88af73b
commit 2f20cf476b
7 changed files with 144 additions and 127 deletions

View file

@ -41,7 +41,7 @@ export default function () {
r: round(Math.max(0, fsStats.rIO_sec ?? 0)), r: round(Math.max(0, fsStats.rIO_sec ?? 0)),
w: round(Math.max(0, fsStats.wIO_sec ?? 0)), w: round(Math.max(0, fsStats.wIO_sec ?? 0)),
}, },
meilisearch: meilisearchStats meilisearch: meilisearchStats,
}; };
ev.emit("serverStats", stats); ev.emit("serverStats", stats);
log.unshift(stats); log.unshift(stats);

View file

@ -1,4 +1,4 @@
import {Health, MeiliSearch, Stats} from 'meilisearch'; import {Health, MeiliSearch, Stats} from "meilisearch";
import {dbLogger} from "./logger.js"; import {dbLogger} from "./logger.js";
import config from "@/config/index.js"; import config from "@/config/index.js";
@ -7,13 +7,15 @@ import * as url from "url";
import {User} from "@/models/entities/user.js"; import {User} from "@/models/entities/user.js";
import {Users} from "@/models/index.js"; import {Users} from "@/models/index.js";
const logger = dbLogger.createSubLogger("meilisearch", "gray", false); const logger = dbLogger.createSubLogger("meilisearch", "gray", false);
logger.info("Connecting to MeiliSearch"); logger.info("Connecting to MeiliSearch");
const hasConfig = const hasConfig =
config.meilisearch && (config.meilisearch.host || config.meilisearch.port || config.meilisearch.apiKey); config.meilisearch &&
(config.meilisearch.host ||
config.meilisearch.port ||
config.meilisearch.apiKey);
const host = hasConfig ? config.meilisearch.host ?? "localhost" : ""; const host = hasConfig ? config.meilisearch.host ?? "localhost" : "";
const port = hasConfig ? config.meilisearch.port ?? 7700 : 0; const port = hasConfig ? config.meilisearch.port ?? 7700 : 0;
@ -22,13 +24,28 @@ const auth = hasConfig ? config.meilisearch.apiKey ?? "" : "";
const client: MeiliSearch = new MeiliSearch({ const client: MeiliSearch = new MeiliSearch({
host: `http://${host}:${port}`, host: `http://${host}:${port}`,
apiKey: auth, apiKey: auth,
}) });
const posts = client.index('posts'); const posts = client.index("posts");
posts.updateSearchableAttributes(['text']).catch((e) => logger.error(`Setting searchable attr failed, searches won't work: ${e}`)); posts
.updateSearchableAttributes(["text"])
.catch((e) =>
logger.error(`Setting searchable attr failed, searches won't work: ${e}`),
);
posts.updateFilterableAttributes(["userName", "userHost", "mediaAttachment", "createdAt"]).catch((e) => logger.error(`Setting filterable attr failed, advanced searches won't work: ${e}`)); posts
.updateFilterableAttributes([
"userName",
"userHost",
"mediaAttachment",
"createdAt",
])
.catch((e) =>
logger.error(
`Setting filterable attr failed, advanced searches won't work: ${e}`,
),
);
logger.info("Connected to MeiliSearch"); logger.info("Connected to MeiliSearch");
@ -40,128 +57,129 @@ export type MeilisearchNote = {
userName: string; userName: string;
channelId: string; channelId: string;
mediaAttachment: string; mediaAttachment: string;
createdAt: number createdAt: number;
} };
export default hasConfig ? { export default hasConfig
search: (query : string, limit : number, offset : number) => { ? {
search: (query: string, limit: number, offset: number) => {
/// Advanced search syntax
/// from:user => filter by user + optional domain
/// has:image/video/audio/text/file => filter by attachment types
/// domain:domain.com => filter by domain
/// before:Date => show posts made before Date
/// after: Date => show posts made after Date
/// Advanced search syntax let constructedFilters: string[] = [];
/// from:user => filter by user + optional domain
/// has:image/video/audio/text/file => filter by attachment types
/// domain:domain.com => filter by domain
/// before:Date => show posts made before Date
/// after: Date => show posts made after Date
let splitSearch = query.split(" ");
let constructedFilters: string[] = []; // Detect search operators and remove them from the actual query
splitSearch = splitSearch.filter((term) => {
let splitSearch = query.split(" "); if (term.startsWith("has:")) {
let fileType = term.slice(4);
// Detect search operators and remove them from the actual query constructedFilters.push(`mediaAttachment = "${fileType}"`);
splitSearch = splitSearch.filter(term => { return false;
if (term.startsWith("has:")) { } else if (term.startsWith("from:")) {
let fileType = term.slice(4); let user = term.slice(5);
constructedFilters.push(`mediaAttachment = "${fileType}"`) constructedFilters.push(`userName = ${user}`);
return false; return false;
} else if (term.startsWith("from:")) { } else if (term.startsWith("domain:")) {
let user = term.slice(5); let domain = term.slice(7);
constructedFilters.push(`userName = ${user}`) constructedFilters.push(`userHost = ${domain}`);
return false; return false;
} else if (term.startsWith("domain:")) { } else if (term.startsWith("after:")) {
let domain = term.slice(7); let timestamp = term.slice(6);
constructedFilters.push(`userHost = ${domain}`) // Try to parse the timestamp as JavaScript Date
return false; let date = Date.parse(timestamp);
} else if (term.startsWith("after:")) { if (isNaN(date)) return false;
let timestamp = term.slice(6); constructedFilters.push(`createdAt > ${date}`);
// Try to parse the timestamp as JavaScript Date return false;
let date = Date.parse(timestamp); } else if (term.startsWith("before:")) {
if (isNaN(date)) return false; let timestamp = term.slice(7);
constructedFilters.push(`createdAt > ${date}`) // Try to parse the timestamp as JavaScript Date
return false; let date = Date.parse(timestamp);
} else if (term.startsWith("before:")) { if (isNaN(date)) return false;
let timestamp = term.slice(7); constructedFilters.push(`createdAt < ${date}`);
// Try to parse the timestamp as JavaScript Date return false;
let date = Date.parse(timestamp);
if (isNaN(date)) return false;
constructedFilters.push(`createdAt < ${date}`)
return false;
}
return true;
})
logger.info(`Searching for ${splitSearch.join(" ")}`);
logger.info(`Limit: ${limit}`);
logger.info(`Offset: ${offset}`);
logger.info(`Filters: ${constructedFilters}`)
return posts.search(splitSearch.join(" "), {
limit: limit,
offset: offset,
filter: constructedFilters
});
},
ingestNote: async (ingestNotes: Note | Note[]) => {
if (ingestNotes instanceof Note) {
ingestNotes = [ingestNotes];
}
let indexingBatch: MeilisearchNote[] = [];
for (let note of ingestNotes) {
if (note.user === undefined) {
let user = await Users.findOne({
where: {
id: note.userId
}
});
note.user = user;
}
let attachmentType = "";
if (note.attachedFileTypes.length > 0) {
attachmentType = note.attachedFileTypes[0].split("/")[0];
switch (attachmentType) {
case "image":
case "video":
case "audio":
case "text":
break;
default:
attachmentType = "file"
break
} }
return true;
});
logger.info(`Searching for ${splitSearch.join(" ")}`);
logger.info(`Limit: ${limit}`);
logger.info(`Offset: ${offset}`);
logger.info(`Filters: ${constructedFilters}`);
return posts.search(splitSearch.join(" "), {
limit: limit,
offset: offset,
filter: constructedFilters,
});
},
ingestNote: async (ingestNotes: Note | Note[]) => {
if (ingestNotes instanceof Note) {
ingestNotes = [ingestNotes];
} }
indexingBatch.push(<MeilisearchNote>{ let indexingBatch: MeilisearchNote[] = [];
for (let note of ingestNotes) {
if (note.user === undefined) {
let user = await Users.findOne({
where: {
id: note.userId,
},
});
note.user = user;
}
let attachmentType = "";
if (note.attachedFileTypes.length > 0) {
attachmentType = note.attachedFileTypes[0].split("/")[0];
switch (attachmentType) {
case "image":
case "video":
case "audio":
case "text":
break;
default:
attachmentType = "file";
break;
}
}
indexingBatch.push(<MeilisearchNote>{
id: note.id.toString(), id: note.id.toString(),
text: note.text ? note.text : "", text: note.text ? note.text : "",
userId: note.userId, userId: note.userId,
userHost: note.userHost !== "" ? note.userHost : url.parse(config.host).host, userHost:
note.userHost !== ""
? note.userHost
: url.parse(config.host).host,
channelId: note.channelId ? note.channelId : "", channelId: note.channelId ? note.channelId : "",
mediaAttachment: attachmentType, mediaAttachment: attachmentType,
userName: note.user?.username ?? "UNKNOWN", userName: note.user?.username ?? "UNKNOWN",
createdAt: note.createdAt.getTime() / 1000 // division by 1000 is necessary because Node returns in ms-accuracy createdAt: note.createdAt.getTime() / 1000, // division by 1000 is necessary because Node returns in ms-accuracy
} });
) }
}
let indexingIDs = indexingBatch.map(note => note.id); let indexingIDs = indexingBatch.map((note) => note.id);
return posts.addDocuments(indexingBatch, { return posts.addDocuments(indexingBatch, {
primaryKey: "id" primaryKey: "id",
}); });
}, },
serverStats: async () => { serverStats: async () => {
let health : Health = await client.health(); let health: Health = await client.health();
let stats: Stats = await client.getStats(); let stats: Stats = await client.getStats();
return { return {
health: health.status, health: health.status,
size: stats.databaseSize, size: stats.databaseSize,
indexed_count: stats.indexes["posts"].numberOfDocuments indexed_count: stats.indexes["posts"].numberOfDocuments,
} };
},
} }
} : null; : null;

View file

@ -3,7 +3,7 @@ import type Bull from "bull";
import { queueLogger } from "../../logger.js"; import { queueLogger } from "../../logger.js";
import { Notes } from "@/models/index.js"; import { Notes } from "@/models/index.js";
import { MoreThan } from "typeorm"; import { MoreThan } from "typeorm";
import { index } from "@/services/note/create.js"; import {index} from "@/services/note/create.js";
import {Note} from "@/models/entities/note.js"; import {Note} from "@/models/entities/note.js";
import meilisearch from "../../../db/meilisearch.js"; import meilisearch from "../../../db/meilisearch.js";
@ -33,13 +33,13 @@ export default async function indexAllNotes(
try { try {
notes = await Notes.find({ notes = await Notes.find({
where: { where: {
...(cursor ? { id: MoreThan(cursor) } : {}), ...(cursor ? {id: MoreThan(cursor)} : {}),
}, },
take: take, take: take,
order: { order: {
id: 1, id: 1,
}, },
relations: ["user"] relations: ["user"],
}); });
} catch (e) { } catch (e) {
logger.error(`Failed to query notes ${e}`); logger.error(`Failed to query notes ${e}`);
@ -62,7 +62,7 @@ export default async function indexAllNotes(
const chunk = notes.slice(i, i + batch); const chunk = notes.slice(i, i + batch);
if (meilisearch) { if (meilisearch) {
await meilisearch.ingestNote(chunk) await meilisearch.ingestNote(chunk);
} }
await Promise.all(chunk.map((note) => index(note, true))); await Promise.all(chunk.map((note) => index(note, true)));

View file

@ -172,7 +172,7 @@ export default define(meta, paramDef, async (ps, me) => {
} }
return found; return found;
} else if(meilisearch) { } else if (meilisearch) {
let start = 0; let start = 0;
const chunkSize = 100; const chunkSize = 100;
@ -236,7 +236,6 @@ export default define(meta, paramDef, async (ps, me) => {
} }
return found; return found;
} else { } else {
const userQuery = const userQuery =
ps.userId != null ps.userId != null

View file

@ -34,8 +34,7 @@ export default define(meta, paramDef, async () => {
total: fsStats[0].size, total: fsStats[0].size,
used: fsStats[0].used, used: fsStats[0].used,
}, },
meilisearch: meilisearchStats meilisearch: meilisearchStats,
}; };
}); });

View file

@ -6,12 +6,13 @@ export async function search() {
const { canceled, result: query } = await os.inputText({ const { canceled, result: query } = await os.inputText({
title: i18n.ts.search, title: i18n.ts.search,
placeholder: "Enter search terms...", placeholder: "Enter search terms...",
text: "Advanced search operators\n" + text:
"Advanced search operators\n" +
"from:user => filter by user\n" + "from:user => filter by user\n" +
"has:image/video/audio/text/file => filter by attachment types\n" + "has:image/video/audio/text/file => filter by attachment types\n" +
"domain:domain.com => filter by domain\n" + "domain:domain.com => filter by domain\n" +
"before:Date => show posts made before Date\n" + "before:Date => show posts made before Date\n" +
"after:Date => show posts made after Date" "after:Date => show posts made after Date",
}); });
if (canceled || query == null || query === "") return; if (canceled || query == null || query === "") return;

View file

@ -61,7 +61,7 @@ import XNet from "./net.vue";
import XCpu from "./cpu.vue"; import XCpu from "./cpu.vue";
import XMemory from "./mem.vue"; import XMemory from "./mem.vue";
import XDisk from "./disk.vue"; import XDisk from "./disk.vue";
import XMeili from "./meilisearch.vue" import XMeili from "./meilisearch.vue";
import MkContainer from "@/components/MkContainer.vue"; import MkContainer from "@/components/MkContainer.vue";
import { GetFormResultType } from "@/scripts/form"; import { GetFormResultType } from "@/scripts/form";
import * as os from "@/os"; import * as os from "@/os";