mirror of
https://iceshrimp.dev/limepotato/jormungandr-bite.git
synced 2024-11-13 13:37:31 -07:00
ファイルと画像認識処理の改善 (#5690)
* dimensions制限とリファクタ * comment * 不要な変更削除 * use fromFile など * Add probe-image-size.d.ts * えーCRLFで作るなよ… * Update src/@types/probe-image-size.d.ts Co-Authored-By: Acid Chicken (硫酸鶏) <root@acid-chicken.com> * fix d.ts * Update src/@types/probe-image-size.d.ts Co-Authored-By: Acid Chicken (硫酸鶏) <root@acid-chicken.com> * Update src/@types/probe-image-size.d.ts Co-Authored-By: Acid Chicken (硫酸鶏) <root@acid-chicken.com> * fix Co-authored-by: Acid Chicken (硫酸鶏) <root@acid-chicken.com>
This commit is contained in:
parent
5acee6e996
commit
fc1d683d3d
20 changed files with 469 additions and 154 deletions
5
.imgbotconfig
Normal file
5
.imgbotconfig
Normal file
|
@ -0,0 +1,5 @@
|
||||||
|
{
|
||||||
|
"ignoredFiles": [
|
||||||
|
"test/resources/*"
|
||||||
|
]
|
||||||
|
}
|
|
@ -180,6 +180,7 @@
|
||||||
"portscanner": "2.2.0",
|
"portscanner": "2.2.0",
|
||||||
"postcss-loader": "3.0.0",
|
"postcss-loader": "3.0.0",
|
||||||
"prismjs": "1.18.0",
|
"prismjs": "1.18.0",
|
||||||
|
"probe-image-size": "5.0.0",
|
||||||
"progress-bar-webpack-plugin": "1.12.1",
|
"progress-bar-webpack-plugin": "1.12.1",
|
||||||
"promise-limit": "2.7.0",
|
"promise-limit": "2.7.0",
|
||||||
"promise-sequential": "1.1.1",
|
"promise-sequential": "1.1.1",
|
||||||
|
|
27
src/@types/probe-image-size.d.ts
vendored
Normal file
27
src/@types/probe-image-size.d.ts
vendored
Normal file
|
@ -0,0 +1,27 @@
|
||||||
|
declare module 'probe-image-size' {
|
||||||
|
import { ReadStream } from 'fs';
|
||||||
|
|
||||||
|
type ProbeOptions = {
|
||||||
|
retries: 1;
|
||||||
|
timeout: 30000;
|
||||||
|
};
|
||||||
|
|
||||||
|
type ProbeResult = {
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
length?: number;
|
||||||
|
type: string;
|
||||||
|
mime: string;
|
||||||
|
wUnits: 'in' | 'mm' | 'cm' | 'pt' | 'pc' | 'px' | 'em' | 'ex';
|
||||||
|
hUnits: 'in' | 'mm' | 'cm' | 'pt' | 'pc' | 'px' | 'em' | 'ex';
|
||||||
|
url?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
function probeImageSize(src: string | ReadStream, options?: ProbeOptions): Promise<ProbeResult>;
|
||||||
|
function probeImageSize(src: string | ReadStream, callback: (err: Error | null, result?: ProbeResult) => void): void;
|
||||||
|
function probeImageSize(src: string | ReadStream, options: ProbeOptions, callback: (err: Error | null, result?: ProbeResult) => void): void;
|
||||||
|
|
||||||
|
namespace probeImageSize {} // Hack
|
||||||
|
|
||||||
|
export = probeImageSize;
|
||||||
|
}
|
|
@ -1,12 +0,0 @@
|
||||||
import * as fs from 'fs';
|
|
||||||
import isSvg from 'is-svg';
|
|
||||||
|
|
||||||
export default function(path: string) {
|
|
||||||
try {
|
|
||||||
const size = fs.statSync(path).size;
|
|
||||||
if (size > 1 * 1024 * 1024) return false;
|
|
||||||
return isSvg(fs.readFileSync(path));
|
|
||||||
} catch {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,31 +0,0 @@
|
||||||
import * as fs from 'fs';
|
|
||||||
import checkSvg from '../misc/check-svg';
|
|
||||||
const FileType = require('file-type');
|
|
||||||
|
|
||||||
export async function detectMine(path: string) {
|
|
||||||
return new Promise<[string, string | null]>((res, rej) => {
|
|
||||||
const readable = fs.createReadStream(path);
|
|
||||||
readable
|
|
||||||
.on('error', rej)
|
|
||||||
.once('data', async (buffer: Buffer) => {
|
|
||||||
readable.destroy();
|
|
||||||
const type = await FileType.fromBuffer(buffer);
|
|
||||||
if (type) {
|
|
||||||
if (type.mime == 'application/xml' && checkSvg(path)) {
|
|
||||||
res(['image/svg+xml', 'svg']);
|
|
||||||
} else {
|
|
||||||
res([type.mime, type.ext]);
|
|
||||||
}
|
|
||||||
} else if (checkSvg(path)) {
|
|
||||||
res(['image/svg+xml', 'svg']);
|
|
||||||
} else {
|
|
||||||
// 種類が同定できなかったら application/octet-stream にする
|
|
||||||
res(['application/octet-stream', null]);
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.on('end', () => {
|
|
||||||
// maybe 0 bytes
|
|
||||||
res(['application/octet-stream', null]);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
|
@ -1,14 +1,14 @@
|
||||||
import { createTemp } from './create-temp';
|
import { createTemp } from './create-temp';
|
||||||
import { downloadUrl } from './donwload-url';
|
import { downloadUrl } from './donwload-url';
|
||||||
import { detectMine } from './detect-mine';
|
import { detectType } from './get-file-info';
|
||||||
|
|
||||||
export async function detectUrlMine(url: string) {
|
export async function detectUrlMime(url: string) {
|
||||||
const [path, cleanup] = await createTemp();
|
const [path, cleanup] = await createTemp();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await downloadUrl(url, path);
|
await downloadUrl(url, path);
|
||||||
const [type] = await detectMine(path);
|
const { mime } = await detectType(path);
|
||||||
return type;
|
return mime;
|
||||||
} finally {
|
} finally {
|
||||||
cleanup();
|
cleanup();
|
||||||
}
|
}
|
201
src/misc/get-file-info.ts
Normal file
201
src/misc/get-file-info.ts
Normal file
|
@ -0,0 +1,201 @@
|
||||||
|
import * as fs from 'fs';
|
||||||
|
import * as crypto from 'crypto';
|
||||||
|
import * as fileType from 'file-type';
|
||||||
|
import isSvg from 'is-svg';
|
||||||
|
import * as probeImageSize from 'probe-image-size';
|
||||||
|
import * as sharp from 'sharp';
|
||||||
|
|
||||||
|
export type FileInfo = {
|
||||||
|
size: number;
|
||||||
|
md5: string;
|
||||||
|
type: {
|
||||||
|
mime: string;
|
||||||
|
ext: string | null;
|
||||||
|
};
|
||||||
|
width?: number;
|
||||||
|
height?: number;
|
||||||
|
avgColor?: number[];
|
||||||
|
warnings: string[];
|
||||||
|
};
|
||||||
|
|
||||||
|
const TYPE_OCTET_STREAM = {
|
||||||
|
mime: 'application/octet-stream',
|
||||||
|
ext: null
|
||||||
|
};
|
||||||
|
|
||||||
|
const TYPE_SVG = {
|
||||||
|
mime: 'image/svg+xml',
|
||||||
|
ext: 'svg'
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get file information
|
||||||
|
*/
|
||||||
|
export async function getFileInfo(path: string): Promise<FileInfo> {
|
||||||
|
const warnings = [] as string[];
|
||||||
|
|
||||||
|
const size = await getFileSize(path);
|
||||||
|
const md5 = await calcHash(path);
|
||||||
|
|
||||||
|
let type = await detectType(path);
|
||||||
|
|
||||||
|
// image dimensions
|
||||||
|
let width: number | undefined;
|
||||||
|
let height: number | undefined;
|
||||||
|
|
||||||
|
if (['image/jpeg', 'image/gif', 'image/png', 'image/apng', 'image/webp', 'image/bmp', 'image/tiff', 'image/svg+xml', 'image/vnd.adobe.photoshop'].includes(type.mime)) {
|
||||||
|
const imageSize = await detectImageSize(path).catch(e => {
|
||||||
|
warnings.push(`detectImageSize failed: ${e}`);
|
||||||
|
return undefined;
|
||||||
|
});
|
||||||
|
|
||||||
|
// うまく判定できない画像は octet-stream にする
|
||||||
|
if (!imageSize) {
|
||||||
|
warnings.push(`cannot detect image dimensions`);
|
||||||
|
type = TYPE_OCTET_STREAM;
|
||||||
|
} else if (imageSize.wUnits === 'px') {
|
||||||
|
width = imageSize.width;
|
||||||
|
height = imageSize.height;
|
||||||
|
|
||||||
|
// 制限を超えている画像は octet-stream にする
|
||||||
|
if (imageSize.width > 16383 || imageSize.height > 16383) {
|
||||||
|
warnings.push(`image dimensions exceeds limits`);
|
||||||
|
type = TYPE_OCTET_STREAM;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
warnings.push(`unsupported unit type: ${imageSize.wUnits}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// average color
|
||||||
|
let avgColor: number[] | undefined;
|
||||||
|
|
||||||
|
if (['image/jpeg', 'image/gif', 'image/png', 'image/apng', 'image/webp', 'image/svg+xml'].includes(type.mime)) {
|
||||||
|
avgColor = await calcAvgColor(path).catch(e => {
|
||||||
|
warnings.push(`calcAvgColor failed: ${e}`);
|
||||||
|
return undefined;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
size,
|
||||||
|
md5,
|
||||||
|
type,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
avgColor,
|
||||||
|
warnings,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Detect MIME Type and extension
|
||||||
|
*/
|
||||||
|
export async function detectType(path: string) {
|
||||||
|
// Check 0 byte
|
||||||
|
const fileSize = await getFileSize(path);
|
||||||
|
if (fileSize === 0) {
|
||||||
|
return TYPE_OCTET_STREAM;
|
||||||
|
}
|
||||||
|
|
||||||
|
const type = await fileType.fromFile(path);
|
||||||
|
|
||||||
|
if (type) {
|
||||||
|
// XMLはSVGかもしれない
|
||||||
|
if (type.mime === 'application/xml' && await checkSvg(path)) {
|
||||||
|
return TYPE_SVG;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
mime: type.mime,
|
||||||
|
ext: type.ext
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// 種類が不明でもSVGかもしれない
|
||||||
|
if (await checkSvg(path)) {
|
||||||
|
return TYPE_SVG;
|
||||||
|
}
|
||||||
|
|
||||||
|
// それでも種類が不明なら application/octet-stream にする
|
||||||
|
return TYPE_OCTET_STREAM;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check the file is SVG or not
|
||||||
|
*/
|
||||||
|
export async function checkSvg(path: string) {
|
||||||
|
try {
|
||||||
|
const size = await getFileSize(path);
|
||||||
|
if (size > 1 * 1024 * 1024) return false;
|
||||||
|
return isSvg(fs.readFileSync(path));
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get file size
|
||||||
|
*/
|
||||||
|
export async function getFileSize(path: string): Promise<number> {
|
||||||
|
return new Promise<number>((res, rej) => {
|
||||||
|
fs.stat(path, (err, stats) => {
|
||||||
|
if (err) return rej(err);
|
||||||
|
res(stats.size);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculate MD5 hash
|
||||||
|
*/
|
||||||
|
async function calcHash(path: string): Promise<string> {
|
||||||
|
return new Promise<string>((res, rej) => {
|
||||||
|
const readable = fs.createReadStream(path);
|
||||||
|
const hash = crypto.createHash('md5');
|
||||||
|
const chunks: Buffer[] = [];
|
||||||
|
readable
|
||||||
|
.on('error', rej)
|
||||||
|
.pipe(hash)
|
||||||
|
.on('error', rej)
|
||||||
|
.on('data', chunk => chunks.push(chunk))
|
||||||
|
.on('end', () => {
|
||||||
|
const buffer = Buffer.concat(chunks);
|
||||||
|
res(buffer.toString('hex'));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Detect dimensions of image
|
||||||
|
*/
|
||||||
|
async function detectImageSize(path: string): Promise<{
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
wUnits: string;
|
||||||
|
hUnits: string;
|
||||||
|
}> {
|
||||||
|
const readable = fs.createReadStream(path);
|
||||||
|
const imageSize = await probeImageSize(readable);
|
||||||
|
readable.destroy();
|
||||||
|
return imageSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculate average color of image
|
||||||
|
*/
|
||||||
|
async function calcAvgColor(path: string): Promise<number[]> {
|
||||||
|
const img = sharp(path);
|
||||||
|
|
||||||
|
const info = await (img as any).stats();
|
||||||
|
|
||||||
|
if (info.isOpaque) {
|
||||||
|
const r = Math.round(info.channels[0].mean);
|
||||||
|
const g = Math.round(info.channels[1].mean);
|
||||||
|
const b = Math.round(info.channels[2].mean);
|
||||||
|
|
||||||
|
return [r, g, b];
|
||||||
|
} else {
|
||||||
|
return [255, 255, 255];
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,6 +1,6 @@
|
||||||
import $ from 'cafy';
|
import $ from 'cafy';
|
||||||
import define from '../../../define';
|
import define from '../../../define';
|
||||||
import { detectUrlMine } from '../../../../../misc/detect-url-mine';
|
import { detectUrlMime } from '../../../../../misc/detect-url-mime';
|
||||||
import { Emojis } from '../../../../../models';
|
import { Emojis } from '../../../../../models';
|
||||||
import { genId } from '../../../../../misc/gen-id';
|
import { genId } from '../../../../../misc/gen-id';
|
||||||
import { getConnection } from 'typeorm';
|
import { getConnection } from 'typeorm';
|
||||||
|
@ -46,7 +46,7 @@ export const meta = {
|
||||||
};
|
};
|
||||||
|
|
||||||
export default define(meta, async (ps, me) => {
|
export default define(meta, async (ps, me) => {
|
||||||
const type = await detectUrlMine(ps.url);
|
const type = await detectUrlMime(ps.url);
|
||||||
|
|
||||||
const exists = await Emojis.findOne({
|
const exists = await Emojis.findOne({
|
||||||
name: ps.name,
|
name: ps.name,
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
import $ from 'cafy';
|
import $ from 'cafy';
|
||||||
import define from '../../../define';
|
import define from '../../../define';
|
||||||
import { detectUrlMine } from '../../../../../misc/detect-url-mine';
|
import { detectUrlMime } from '../../../../../misc/detect-url-mime';
|
||||||
import { ID } from '../../../../../misc/cafy-id';
|
import { ID } from '../../../../../misc/cafy-id';
|
||||||
import { Emojis } from '../../../../../models';
|
import { Emojis } from '../../../../../models';
|
||||||
import { getConnection } from 'typeorm';
|
import { getConnection } from 'typeorm';
|
||||||
|
@ -52,7 +52,7 @@ export default define(meta, async (ps) => {
|
||||||
|
|
||||||
if (emoji == null) throw new ApiError(meta.errors.noSuchEmoji);
|
if (emoji == null) throw new ApiError(meta.errors.noSuchEmoji);
|
||||||
|
|
||||||
const type = await detectUrlMine(ps.url);
|
const type = await detectUrlMime(ps.url);
|
||||||
|
|
||||||
await Emojis.update(emoji.id, {
|
await Emojis.update(emoji.id, {
|
||||||
updatedAt: new Date(),
|
updatedAt: new Date(),
|
||||||
|
|
|
@ -8,7 +8,7 @@ import { contentDisposition } from '../../misc/content-disposition';
|
||||||
import { DriveFiles } from '../../models';
|
import { DriveFiles } from '../../models';
|
||||||
import { InternalStorage } from '../../services/drive/internal-storage';
|
import { InternalStorage } from '../../services/drive/internal-storage';
|
||||||
import { downloadUrl } from '../../misc/donwload-url';
|
import { downloadUrl } from '../../misc/donwload-url';
|
||||||
import { detectMine } from '../../misc/detect-mine';
|
import { detectType } from '../../misc/get-file-info';
|
||||||
import { convertToJpeg, convertToPng } from '../../services/drive/image-processor';
|
import { convertToJpeg, convertToPng } from '../../services/drive/image-processor';
|
||||||
import { GenerateVideoThumbnail } from '../../services/drive/generate-video-thumbnail';
|
import { GenerateVideoThumbnail } from '../../services/drive/generate-video-thumbnail';
|
||||||
|
|
||||||
|
@ -52,15 +52,15 @@ export default async function(ctx: Koa.Context) {
|
||||||
try {
|
try {
|
||||||
await downloadUrl(file.uri, path);
|
await downloadUrl(file.uri, path);
|
||||||
|
|
||||||
const [type, ext] = await detectMine(path);
|
const { mime, ext } = await detectType(path);
|
||||||
|
|
||||||
const convertFile = async () => {
|
const convertFile = async () => {
|
||||||
if (isThumbnail) {
|
if (isThumbnail) {
|
||||||
if (['image/jpeg', 'image/webp'].includes(type)) {
|
if (['image/jpeg', 'image/webp'].includes(mime)) {
|
||||||
return await convertToJpeg(path, 498, 280);
|
return await convertToJpeg(path, 498, 280);
|
||||||
} else if (['image/png'].includes(type)) {
|
} else if (['image/png'].includes(mime)) {
|
||||||
return await convertToPng(path, 498, 280);
|
return await convertToPng(path, 498, 280);
|
||||||
} else if (type.startsWith('video/')) {
|
} else if (mime.startsWith('video/')) {
|
||||||
return await GenerateVideoThumbnail(path);
|
return await GenerateVideoThumbnail(path);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -68,7 +68,7 @@ export default async function(ctx: Koa.Context) {
|
||||||
return {
|
return {
|
||||||
data: fs.readFileSync(path),
|
data: fs.readFileSync(path),
|
||||||
ext,
|
ext,
|
||||||
type,
|
type: mime,
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -4,7 +4,7 @@ import { serverLogger } from '..';
|
||||||
import { IImage, convertToPng, convertToJpeg } from '../../services/drive/image-processor';
|
import { IImage, convertToPng, convertToJpeg } from '../../services/drive/image-processor';
|
||||||
import { createTemp } from '../../misc/create-temp';
|
import { createTemp } from '../../misc/create-temp';
|
||||||
import { downloadUrl } from '../../misc/donwload-url';
|
import { downloadUrl } from '../../misc/donwload-url';
|
||||||
import { detectMine } from '../../misc/detect-mine';
|
import { detectType } from '../../misc/get-file-info';
|
||||||
|
|
||||||
export async function proxyMedia(ctx: Koa.Context) {
|
export async function proxyMedia(ctx: Koa.Context) {
|
||||||
const url = 'url' in ctx.query ? ctx.query.url : 'https://' + ctx.params.url;
|
const url = 'url' in ctx.query ? ctx.query.url : 'https://' + ctx.params.url;
|
||||||
|
@ -15,21 +15,21 @@ export async function proxyMedia(ctx: Koa.Context) {
|
||||||
try {
|
try {
|
||||||
await downloadUrl(url, path);
|
await downloadUrl(url, path);
|
||||||
|
|
||||||
const [type, ext] = await detectMine(path);
|
const { mime, ext } = await detectType(path);
|
||||||
|
|
||||||
if (!type.startsWith('image/')) throw 403;
|
if (!mime.startsWith('image/')) throw 403;
|
||||||
|
|
||||||
let image: IImage;
|
let image: IImage;
|
||||||
|
|
||||||
if ('static' in ctx.query && ['image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng'].includes(type)) {
|
if ('static' in ctx.query && ['image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng'].includes(mime)) {
|
||||||
image = await convertToPng(path, 498, 280);
|
image = await convertToPng(path, 498, 280);
|
||||||
} else if ('preview' in ctx.query && ['image/jpeg', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng'].includes(type)) {
|
} else if ('preview' in ctx.query && ['image/jpeg', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng'].includes(mime)) {
|
||||||
image = await convertToJpeg(path, 200, 200);
|
image = await convertToJpeg(path, 200, 200);
|
||||||
} else {
|
} else {
|
||||||
image = {
|
image = {
|
||||||
data: fs.readFileSync(path),
|
data: fs.readFileSync(path),
|
||||||
ext,
|
ext,
|
||||||
type,
|
type: mime,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,9 +1,6 @@
|
||||||
import { Buffer } from 'buffer';
|
|
||||||
import * as fs from 'fs';
|
import * as fs from 'fs';
|
||||||
|
|
||||||
import * as crypto from 'crypto';
|
|
||||||
import { v4 as uuid } from 'uuid';
|
import { v4 as uuid } from 'uuid';
|
||||||
import * as sharp from 'sharp';
|
|
||||||
|
|
||||||
import { publishMainStream, publishDriveStream } from '../stream';
|
import { publishMainStream, publishDriveStream } from '../stream';
|
||||||
import { deleteFile } from './delete-file';
|
import { deleteFile } from './delete-file';
|
||||||
|
@ -12,7 +9,7 @@ import { GenerateVideoThumbnail } from './generate-video-thumbnail';
|
||||||
import { driveLogger } from './logger';
|
import { driveLogger } from './logger';
|
||||||
import { IImage, convertToJpeg, convertToWebp, convertToPng } from './image-processor';
|
import { IImage, convertToJpeg, convertToWebp, convertToPng } from './image-processor';
|
||||||
import { contentDisposition } from '../../misc/content-disposition';
|
import { contentDisposition } from '../../misc/content-disposition';
|
||||||
import { detectMine } from '../../misc/detect-mine';
|
import { getFileInfo } from '../../misc/get-file-info';
|
||||||
import { DriveFiles, DriveFolders, Users, Instances, UserProfiles } from '../../models';
|
import { DriveFiles, DriveFolders, Users, Instances, UserProfiles } from '../../models';
|
||||||
import { InternalStorage } from './internal-storage';
|
import { InternalStorage } from './internal-storage';
|
||||||
import { DriveFile } from '../../models/entities/drive-file';
|
import { DriveFile } from '../../models/entities/drive-file';
|
||||||
|
@ -271,41 +268,16 @@ export default async function(
|
||||||
uri: string | null = null,
|
uri: string | null = null,
|
||||||
sensitive: boolean | null = null
|
sensitive: boolean | null = null
|
||||||
): Promise<DriveFile> {
|
): Promise<DriveFile> {
|
||||||
// Calc md5 hash
|
const info = await getFileInfo(path);
|
||||||
const calcHash = new Promise<string>((res, rej) => {
|
logger.info(`${JSON.stringify(info)}`);
|
||||||
const readable = fs.createReadStream(path);
|
|
||||||
const hash = crypto.createHash('md5');
|
|
||||||
const chunks: Buffer[] = [];
|
|
||||||
readable
|
|
||||||
.on('error', rej)
|
|
||||||
.pipe(hash)
|
|
||||||
.on('error', rej)
|
|
||||||
.on('data', chunk => chunks.push(chunk))
|
|
||||||
.on('end', () => {
|
|
||||||
const buffer = Buffer.concat(chunks);
|
|
||||||
res(buffer.toString('hex'));
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
// Get file size
|
|
||||||
const getFileSize = new Promise<number>((res, rej) => {
|
|
||||||
fs.stat(path, (err, stats) => {
|
|
||||||
if (err) return rej(err);
|
|
||||||
res(stats.size);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
const [hash, [mime, ext], size] = await Promise.all([calcHash, detectMine(path), getFileSize]);
|
|
||||||
|
|
||||||
logger.info(`hash: ${hash}, mime: ${mime}, ext: ${ext}, size: ${size}`);
|
|
||||||
|
|
||||||
// detect name
|
// detect name
|
||||||
const detectedName = name || (ext ? `untitled.${ext}` : 'untitled');
|
const detectedName = name || (info.type.ext ? `untitled.${info.type.ext}` : 'untitled');
|
||||||
|
|
||||||
if (!force) {
|
if (!force) {
|
||||||
// Check if there is a file with the same hash
|
// Check if there is a file with the same hash
|
||||||
const much = await DriveFiles.findOne({
|
const much = await DriveFiles.findOne({
|
||||||
md5: hash,
|
md5: info.md5,
|
||||||
userId: user.id,
|
userId: user.id,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -325,7 +297,7 @@ export default async function(
|
||||||
logger.debug(`drive usage is ${usage} (max: ${driveCapacity})`);
|
logger.debug(`drive usage is ${usage} (max: ${driveCapacity})`);
|
||||||
|
|
||||||
// If usage limit exceeded
|
// If usage limit exceeded
|
||||||
if (usage + size > driveCapacity) {
|
if (usage + info.size > driveCapacity) {
|
||||||
if (Users.isLocalUser(user)) {
|
if (Users.isLocalUser(user)) {
|
||||||
throw new Error('no-free-space');
|
throw new Error('no-free-space');
|
||||||
} else {
|
} else {
|
||||||
|
@ -351,57 +323,24 @@ export default async function(
|
||||||
return driveFolder;
|
return driveFolder;
|
||||||
};
|
};
|
||||||
|
|
||||||
const properties: {[key: string]: any} = {};
|
const properties: {
|
||||||
|
width?: number;
|
||||||
|
height?: number;
|
||||||
|
avgColor?: string;
|
||||||
|
} = {};
|
||||||
|
|
||||||
let propPromises: Promise<void>[] = [];
|
if (info.width) {
|
||||||
|
properties['width'] = info.width;
|
||||||
|
properties['height'] = info.height;
|
||||||
|
}
|
||||||
|
|
||||||
const isImage = ['image/jpeg', 'image/gif', 'image/png', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/svg+xml'].includes(mime);
|
if (info.avgColor) {
|
||||||
|
properties['avgColor'] = `rgb(${info.avgColor.join(',')}`;
|
||||||
if (isImage) {
|
|
||||||
const img = sharp(path);
|
|
||||||
|
|
||||||
// Calc width and height
|
|
||||||
const calcWh = async () => {
|
|
||||||
logger.debug('calculating image width and height...');
|
|
||||||
|
|
||||||
// Calculate width and height
|
|
||||||
const meta = await img.metadata();
|
|
||||||
|
|
||||||
logger.debug(`image width and height is calculated: ${meta.width}, ${meta.height}`);
|
|
||||||
|
|
||||||
properties['width'] = meta.width;
|
|
||||||
properties['height'] = meta.height;
|
|
||||||
};
|
|
||||||
|
|
||||||
// Calc average color
|
|
||||||
const calcAvg = async () => {
|
|
||||||
logger.debug('calculating average color...');
|
|
||||||
|
|
||||||
try {
|
|
||||||
const info = await img.stats();
|
|
||||||
|
|
||||||
if (info.isOpaque) {
|
|
||||||
const r = Math.round(info.channels[0].mean);
|
|
||||||
const g = Math.round(info.channels[1].mean);
|
|
||||||
const b = Math.round(info.channels[2].mean);
|
|
||||||
|
|
||||||
logger.debug(`average color is calculated: ${r}, ${g}, ${b}`);
|
|
||||||
|
|
||||||
properties['avgColor'] = `rgb(${r},${g},${b})`;
|
|
||||||
} else {
|
|
||||||
logger.debug(`this image is not opaque so average color is 255, 255, 255`);
|
|
||||||
|
|
||||||
properties['avgColor'] = `rgb(255,255,255)`;
|
|
||||||
}
|
|
||||||
} catch (e) { }
|
|
||||||
};
|
|
||||||
|
|
||||||
propPromises = [calcWh(), calcAvg()];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const profile = await UserProfiles.findOne(user.id);
|
const profile = await UserProfiles.findOne(user.id);
|
||||||
|
|
||||||
const [folder] = await Promise.all([fetchFolder(), Promise.all(propPromises)]);
|
const folder = await fetchFolder();
|
||||||
|
|
||||||
let file = new DriveFile();
|
let file = new DriveFile();
|
||||||
file.id = genId();
|
file.id = genId();
|
||||||
|
@ -436,9 +375,9 @@ export default async function(
|
||||||
if (isLink) {
|
if (isLink) {
|
||||||
try {
|
try {
|
||||||
file.size = 0;
|
file.size = 0;
|
||||||
file.md5 = hash;
|
file.md5 = info.md5;
|
||||||
file.name = detectedName;
|
file.name = detectedName;
|
||||||
file.type = mime;
|
file.type = info.type.mime;
|
||||||
file.storedInternal = false;
|
file.storedInternal = false;
|
||||||
|
|
||||||
file = await DriveFiles.save(file);
|
file = await DriveFiles.save(file);
|
||||||
|
@ -457,7 +396,7 @@ export default async function(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
file = await (save(file, path, detectedName, mime, hash, size));
|
file = await (save(file, path, detectedName, info.type.mime, info.md5, info.size));
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.succ(`drive file has been created ${file.id}`);
|
logger.succ(`drive file has been created ${file.id}`);
|
||||||
|
|
152
test/get-file-info.ts
Normal file
152
test/get-file-info.ts
Normal file
|
@ -0,0 +1,152 @@
|
||||||
|
/*
|
||||||
|
* Tests for detection of file information
|
||||||
|
*
|
||||||
|
* How to run the tests:
|
||||||
|
* > TS_NODE_FILES=true npx mocha test/get-file-info.ts --require ts-node/register
|
||||||
|
*
|
||||||
|
* To specify test:
|
||||||
|
* > TS_NODE_FILES=true npx mocha test/get-file-info.ts --require ts-node/register -g 'test name'
|
||||||
|
*/
|
||||||
|
|
||||||
|
import * as assert from 'assert';
|
||||||
|
import { async } from './utils';
|
||||||
|
import { getFileInfo } from '../src/misc/get-file-info';
|
||||||
|
|
||||||
|
describe('Get file info', () => {
|
||||||
|
it('Empty file', async (async () => {
|
||||||
|
const path = `${__dirname}/resources/emptyfile`;
|
||||||
|
const info = await getFileInfo(path);
|
||||||
|
delete info.warnings;
|
||||||
|
assert.deepStrictEqual(info, {
|
||||||
|
size: 0,
|
||||||
|
md5: 'd41d8cd98f00b204e9800998ecf8427e',
|
||||||
|
type: {
|
||||||
|
mime: 'application/octet-stream',
|
||||||
|
ext: null
|
||||||
|
},
|
||||||
|
width: undefined,
|
||||||
|
height: undefined,
|
||||||
|
avgColor: undefined
|
||||||
|
});
|
||||||
|
}));
|
||||||
|
|
||||||
|
it('Generic JPEG', async (async () => {
|
||||||
|
const path = `${__dirname}/resources/Lenna.jpg`;
|
||||||
|
const info = await getFileInfo(path);
|
||||||
|
delete info.warnings;
|
||||||
|
assert.deepStrictEqual(info, {
|
||||||
|
size: 25360,
|
||||||
|
md5: '091b3f259662aa31e2ffef4519951168',
|
||||||
|
type: {
|
||||||
|
mime: 'image/jpeg',
|
||||||
|
ext: 'jpg'
|
||||||
|
},
|
||||||
|
width: 512,
|
||||||
|
height: 512,
|
||||||
|
avgColor: [ 181, 99, 106 ]
|
||||||
|
});
|
||||||
|
}));
|
||||||
|
|
||||||
|
it('Generic APNG', async (async () => {
|
||||||
|
const path = `${__dirname}/resources/anime.png`;
|
||||||
|
const info = await getFileInfo(path);
|
||||||
|
delete info.warnings;
|
||||||
|
assert.deepStrictEqual(info, {
|
||||||
|
size: 1868,
|
||||||
|
md5: '08189c607bea3b952704676bb3c979e0',
|
||||||
|
type: {
|
||||||
|
mime: 'image/apng',
|
||||||
|
ext: 'apng'
|
||||||
|
},
|
||||||
|
width: 256,
|
||||||
|
height: 256,
|
||||||
|
avgColor: [ 249, 253, 250 ]
|
||||||
|
});
|
||||||
|
}));
|
||||||
|
|
||||||
|
it('Generic AGIF', async (async () => {
|
||||||
|
const path = `${__dirname}/resources/anime.gif`;
|
||||||
|
const info = await getFileInfo(path);
|
||||||
|
delete info.warnings;
|
||||||
|
assert.deepStrictEqual(info, {
|
||||||
|
size: 2248,
|
||||||
|
md5: '32c47a11555675d9267aee1a86571e7e',
|
||||||
|
type: {
|
||||||
|
mime: 'image/gif',
|
||||||
|
ext: 'gif'
|
||||||
|
},
|
||||||
|
width: 256,
|
||||||
|
height: 256,
|
||||||
|
avgColor: [ 249, 253, 250 ]
|
||||||
|
});
|
||||||
|
}));
|
||||||
|
|
||||||
|
it('PNG with alpha', async (async () => {
|
||||||
|
const path = `${__dirname}/resources/with-alpha.png`;
|
||||||
|
const info = await getFileInfo(path);
|
||||||
|
delete info.warnings;
|
||||||
|
assert.deepStrictEqual(info, {
|
||||||
|
size: 3772,
|
||||||
|
md5: 'f73535c3e1e27508885b69b10cf6e991',
|
||||||
|
type: {
|
||||||
|
mime: 'image/png',
|
||||||
|
ext: 'png'
|
||||||
|
},
|
||||||
|
width: 256,
|
||||||
|
height: 256,
|
||||||
|
avgColor: [ 255, 255, 255 ]
|
||||||
|
});
|
||||||
|
}));
|
||||||
|
|
||||||
|
it('Generic SVG', async (async () => {
|
||||||
|
const path = `${__dirname}/resources/image.svg`;
|
||||||
|
const info = await getFileInfo(path);
|
||||||
|
delete info.warnings;
|
||||||
|
assert.deepStrictEqual(info, {
|
||||||
|
size: 505,
|
||||||
|
md5: 'b6f52b4b021e7b92cdd04509c7267965',
|
||||||
|
type: {
|
||||||
|
mime: 'image/svg+xml',
|
||||||
|
ext: 'svg'
|
||||||
|
},
|
||||||
|
width: 256,
|
||||||
|
height: 256,
|
||||||
|
avgColor: [ 255, 255, 255 ]
|
||||||
|
});
|
||||||
|
}));
|
||||||
|
|
||||||
|
it('SVG with XML definition', async (async () => {
|
||||||
|
// https://github.com/syuilo/misskey/issues/4413
|
||||||
|
const path = `${__dirname}/resources/with-xml-def.svg`;
|
||||||
|
const info = await getFileInfo(path);
|
||||||
|
delete info.warnings;
|
||||||
|
assert.deepStrictEqual(info, {
|
||||||
|
size: 544,
|
||||||
|
md5: '4b7a346cde9ccbeb267e812567e33397',
|
||||||
|
type: {
|
||||||
|
mime: 'image/svg+xml',
|
||||||
|
ext: 'svg'
|
||||||
|
},
|
||||||
|
width: 256,
|
||||||
|
height: 256,
|
||||||
|
avgColor: [ 255, 255, 255 ]
|
||||||
|
});
|
||||||
|
}));
|
||||||
|
|
||||||
|
it('Dimension limit', async (async () => {
|
||||||
|
const path = `${__dirname}/resources/25000x25000.png`;
|
||||||
|
const info = await getFileInfo(path);
|
||||||
|
delete info.warnings;
|
||||||
|
assert.deepStrictEqual(info, {
|
||||||
|
size: 75933,
|
||||||
|
md5: '268c5dde99e17cf8fe09f1ab3f97df56',
|
||||||
|
type: {
|
||||||
|
mime: 'application/octet-stream', // do not treat as image
|
||||||
|
ext: null
|
||||||
|
},
|
||||||
|
width: 25000,
|
||||||
|
height: 25000,
|
||||||
|
avgColor: undefined
|
||||||
|
});
|
||||||
|
}));
|
||||||
|
});
|
BIN
test/resources/25000x25000.png
(Stored with Git LFS)
Normal file
BIN
test/resources/25000x25000.png
(Stored with Git LFS)
Normal file
Binary file not shown.
BIN
test/resources/anime.gif
(Stored with Git LFS)
Normal file
BIN
test/resources/anime.gif
(Stored with Git LFS)
Normal file
Binary file not shown.
BIN
test/resources/anime.png
(Stored with Git LFS)
Normal file
BIN
test/resources/anime.png
(Stored with Git LFS)
Normal file
Binary file not shown.
0
test/resources/emptyfile
Normal file
0
test/resources/emptyfile
Normal file
BIN
test/resources/with-alpha.png
(Stored with Git LFS)
Normal file
BIN
test/resources/with-alpha.png
(Stored with Git LFS)
Normal file
Binary file not shown.
BIN
test/resources/with-xml-def.svg
(Stored with Git LFS)
Normal file
BIN
test/resources/with-xml-def.svg
(Stored with Git LFS)
Normal file
Binary file not shown.
36
yarn.lock
36
yarn.lock
|
@ -3038,6 +3038,13 @@ debug-fabulous@1.X:
|
||||||
memoizee "0.4.X"
|
memoizee "0.4.X"
|
||||||
object-assign "4.X"
|
object-assign "4.X"
|
||||||
|
|
||||||
|
debug@2, debug@^2.2.0, debug@^2.3.3, debug@^2.5.2:
|
||||||
|
version "2.6.9"
|
||||||
|
resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
|
||||||
|
integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==
|
||||||
|
dependencies:
|
||||||
|
ms "2.0.0"
|
||||||
|
|
||||||
debug@3.1.0, debug@~3.1.0:
|
debug@3.1.0, debug@~3.1.0:
|
||||||
version "3.1.0"
|
version "3.1.0"
|
||||||
resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261"
|
resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261"
|
||||||
|
@ -3059,13 +3066,6 @@ debug@4, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1:
|
||||||
dependencies:
|
dependencies:
|
||||||
ms "^2.1.1"
|
ms "^2.1.1"
|
||||||
|
|
||||||
debug@^2.2.0, debug@^2.3.3, debug@^2.5.2:
|
|
||||||
version "2.6.9"
|
|
||||||
resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
|
|
||||||
integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==
|
|
||||||
dependencies:
|
|
||||||
ms "2.0.0"
|
|
||||||
|
|
||||||
debuglog@^1.0.0:
|
debuglog@^1.0.0:
|
||||||
version "1.0.1"
|
version "1.0.1"
|
||||||
resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492"
|
resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492"
|
||||||
|
@ -3110,7 +3110,7 @@ deep-is@~0.1.3:
|
||||||
resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34"
|
resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34"
|
||||||
integrity sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=
|
integrity sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=
|
||||||
|
|
||||||
deepmerge@^4.2.2:
|
deepmerge@^4.0.0, deepmerge@^4.2.2:
|
||||||
version "4.2.2"
|
version "4.2.2"
|
||||||
resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.2.2.tgz#44d2ea3679b8f4d4ffba33f03d865fc1e7bf4955"
|
resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.2.2.tgz#44d2ea3679b8f4d4ffba33f03d865fc1e7bf4955"
|
||||||
integrity sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==
|
integrity sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==
|
||||||
|
@ -8040,6 +8040,17 @@ prismjs@1.18.0:
|
||||||
optionalDependencies:
|
optionalDependencies:
|
||||||
clipboard "^2.0.0"
|
clipboard "^2.0.0"
|
||||||
|
|
||||||
|
probe-image-size@5.0.0:
|
||||||
|
version "5.0.0"
|
||||||
|
resolved "https://registry.yarnpkg.com/probe-image-size/-/probe-image-size-5.0.0.tgz#1b87d20340ab8fcdb4324ec77fbc8a5f53419878"
|
||||||
|
integrity sha512-V6uBYw5eBc5UVIE7MUZD6Nxg0RYuGDWLDenEn0B1WC6PcTvn1xdQ6HLDDuznefsiExC6rNrCz7mFRBo0f3Xekg==
|
||||||
|
dependencies:
|
||||||
|
deepmerge "^4.0.0"
|
||||||
|
inherits "^2.0.3"
|
||||||
|
next-tick "^1.0.0"
|
||||||
|
request "^2.83.0"
|
||||||
|
stream-parser "~0.3.1"
|
||||||
|
|
||||||
process-nextick-args@^2.0.0, process-nextick-args@~2.0.0:
|
process-nextick-args@^2.0.0, process-nextick-args@~2.0.0:
|
||||||
version "2.0.1"
|
version "2.0.1"
|
||||||
resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"
|
resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"
|
||||||
|
@ -8652,7 +8663,7 @@ request-stats@3.0.0:
|
||||||
http-headers "^3.0.1"
|
http-headers "^3.0.1"
|
||||||
once "^1.4.0"
|
once "^1.4.0"
|
||||||
|
|
||||||
request@2.88.0, request@^2.73.0, request@^2.88.0:
|
request@2.88.0, request@^2.73.0, request@^2.83.0, request@^2.88.0:
|
||||||
version "2.88.0"
|
version "2.88.0"
|
||||||
resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef"
|
resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef"
|
||||||
integrity sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==
|
integrity sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==
|
||||||
|
@ -9356,6 +9367,13 @@ stream-http@^2.7.2:
|
||||||
to-arraybuffer "^1.0.0"
|
to-arraybuffer "^1.0.0"
|
||||||
xtend "^4.0.0"
|
xtend "^4.0.0"
|
||||||
|
|
||||||
|
stream-parser@~0.3.1:
|
||||||
|
version "0.3.1"
|
||||||
|
resolved "https://registry.yarnpkg.com/stream-parser/-/stream-parser-0.3.1.tgz#1618548694420021a1182ff0af1911c129761773"
|
||||||
|
integrity sha1-FhhUhpRCACGhGC/wrxkRwSl2F3M=
|
||||||
|
dependencies:
|
||||||
|
debug "2"
|
||||||
|
|
||||||
stream-shift@^1.0.0:
|
stream-shift@^1.0.0:
|
||||||
version "1.0.0"
|
version "1.0.0"
|
||||||
resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.0.tgz#d5c752825e5367e786f78e18e445ea223a155952"
|
resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.0.tgz#d5c752825e5367e786f78e18e445ea223a155952"
|
||||||
|
|
Loading…
Reference in a new issue