Merge branch 'develop' of https://codeberg.org/calckey/calckey into upstream

.config/example.yml

@@ -121,7 +121,7 @@ redis:
 #   ┌─────────────────────┐
 #───┘ Other configuration └─────────────────────────────────────
 
-# Maximum length of a post (default 3000, max 250000000)
+# Maximum length of a post (default 3000, max 100000)
 #maxNoteLength: 3000
 
 # Maximum length of an image caption (default 1500, max 8192)

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "calckey",
-	"version": "14.0.0-dev77",
+	"version": "14.0.0-dev78",
 	"codename": "aqua",
 	"repository": {
 		"type": "git",

packages/backend/migration/1678426061773-tweak-varchar-length.js

@@ -0,0 +1,10 @@
+export class tweakVarcharLength1678426061773 {
+		name = 'tweakVarcharLength1678426061773'
+
+		async up(queryRunner) {
+			await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "smtpUser" TYPE character varying(1024)`, undefined);
+			await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "smtpPass" TYPE character varying(1024)`, undefined);
+		}
+
+		async down(queryRunner) {}
+}

packages/backend/src/const.ts

@@ -1,7 +1,10 @@
 import config from "@/config/index.js";
-import { DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js";
+import { DB_MAX_NOTE_TEXT_LENGTH, DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js";
 
-export const MAX_NOTE_TEXT_LENGTH = config.maxNoteLength ?? 3000;
+export const MAX_NOTE_TEXT_LENGTH = Math.min(
+	config.maxNoteLength ?? 3000,
+	DB_MAX_NOTE_TEXT_LENGTH,
+);
 export const MAX_CAPTION_TEXT_LENGTH = Math.min(
 	config.maxCaptionLength ?? 1500,
 	DB_MAX_IMAGE_COMMENT_LENGTH,

packages/backend/src/misc/hard-limits.ts

@@ -3,9 +3,13 @@
 /**
  * Maximum note text length that can be stored in DB.
  * Surrogate pairs count as one
- * DEPRECARTED: use const/MAX_NOTE_TEXT_LENGTH instead
+ *
+ * NOTE: this can hypothetically be pushed further
+ * (up to 250000000), but will likely cause truncations
+ * and incompatibilities with other servers,
+ * as well as potential performance issues.
  */
-// export const DB_MAX_NOTE_TEXT_LENGTH = 8192;
+export const DB_MAX_NOTE_TEXT_LENGTH = 100000;
 
 /**
  * Maximum image description length that can be stored in DB.

packages/backend/src/models/entities/meta.ts

@@ -326,13 +326,13 @@ export class Meta {
 	public smtpPort: number | null;
 
 	@Column("varchar", {
-		length: 128,
+		length: 1024,
 		nullable: true,
 	})
 	public smtpUser: string | null;
 
 	@Column("varchar", {
-		length: 128,
+		length: 1024,
 		nullable: true,
 	})
 	public smtpPass: string | null;

packages/backend/src/queue/processors/db/export-notes.ts

@@ -4,7 +4,7 @@ import * as fs from "node:fs";
 import { queueLogger } from "../../logger.js";
 import { addFile } from "@/services/drive/add-file.js";
 import { format as dateFormat } from "date-fns";
-import { Users, Notes, Polls } from "@/models/index.js";
+import { Users, Notes, Polls, DriveFiles } from "@/models/index.js";
 import { MoreThan } from "typeorm";
 import type { Note } from "@/models/entities/note.js";
 import type { Poll } from "@/models/entities/poll.js";
@@ -75,7 +75,7 @@ export async function exportNotes(
 				if (note.hasPoll) {
 					poll = await Polls.findOneByOrFail({ noteId: note.id });
 				}
-				const content = JSON.stringify(serialize(note, poll));
+				const content = JSON.stringify(await serialize(note, poll));
 				const isFirst = exportedNotesCount === 0;
 				await write(isFirst ? content : ",\n" + content);
 				exportedNotesCount++;
@@ -112,15 +112,16 @@ export async function exportNotes(
 	done();
 }
 
-function serialize(
+async function serialize(
 	note: Note,
 	poll: Poll | null = null,
-): Record<string, unknown> {
+): Promise<Record<string, unknown>> {
 	return {
 		id: note.id,
 		text: note.text,
 		createdAt: note.createdAt,
 		fileIds: note.fileIds,
+		files: await DriveFiles.packMany(note.fileIds),
 		replyId: note.replyId,
 		renoteId: note.renoteId,
 		poll: poll,

packages/backend/src/queue/processors/db/import-calckey-post.ts

@@ -3,6 +3,8 @@ import create from "@/services/note/create.js";
 import { Users } from "@/models/index.js";
 import type { DbUserImportMastoPostJobData } from "@/queue/types.js";
 import { queueLogger } from "../../logger.js";
+import { uploadFromUrl } from "@/services/drive/upload-from-url.js";
+import type { DriveFile } from "@/models/entities/drive-file.js";
 import type Bull from "bull";
 
 const logger = queueLogger.createSubLogger("import-calckey-post");
@@ -29,10 +31,25 @@ export async function importCkPost(
 		done();
 		return;
 	}
+	const urls = (post.files || [])
+		.map((x: any) => x.url)
+		.filter((x: String) => x.startsWith("http"));
+	const files: DriveFile[] = [];
+	for (const url of urls) {
+		try {
+			const file = await uploadFromUrl({
+				url: url,
+				user: user,
+			});
+			files.push(file);
+		} catch (e) {
+			logger.error(`Skipped adding file to drive: ${url}`);
+		}
+	}
 	const { text, cw, localOnly, createdAt } = Post.parse(post);
 	const note = await create(user, {
 		createdAt: createdAt,
-		files: undefined,
+		files: files.length == 0 ? undefined : files,
 		poll: undefined,
 		text: text || undefined,
 		reply: null,

packages/backend/src/queue/processors/db/import-masto-post.ts

@@ -6,6 +6,8 @@ import type Bull from "bull";
 import { htmlToMfm } from "@/remote/activitypub/misc/html-to-mfm.js";
 import { resolveNote } from "@/remote/activitypub/models/note.js";
 import { Note } from "@/models/entities/note.js";
+import { uploadFromUrl } from "@/services/drive/upload-from-url.js";
+import type { DriveFile } from "@/models/entities/drive-file.js";
 
 const logger = queueLogger.createSubLogger("import-masto-post");
 
@@ -43,9 +45,25 @@ export async function importMastoPost(
 		throw e;
 	}
 	job.progress(80);
+	const urls = post.object.attachment
+		.map((x: any) => x.url)
+		.filter((x: String) => x.startsWith("http"));
+	const files: DriveFile[] = [];
+	for (const url of urls) {
+		try {
+			const file = await uploadFromUrl({
+				url: url,
+				user: user,
+			});
+			files.push(file);
+		} catch (e) {
+			logger.error(`Skipped adding file to drive: ${url}`);
+		}
+	}
+
 	const note = await create(user, {
 		createdAt: new Date(post.object.published),
-		files: undefined,
+		files: files.length == 0 ? undefined : files,
 		poll: undefined,
 		text: text || undefined,
 		reply,