Merge pull request #1968 from syuilo/object-storage

Object storage support
This commit is contained in:
syuilo 2018-07-24 23:45:19 +09:00 committed by GitHub
commit 8e82e22c9f
45 changed files with 174 additions and 159 deletions

View file

@ -53,6 +53,22 @@ remoteDriveCapacityMb: 8
# Users cannot see remote images when they turn off "Show media from a remote server" setting.
preventCache: false
drive:
storage: 'db'
# OR
# storage: 'object-storage'
# service: 'minio'
# bucket:
# prefix:
# config:
# endPoint:
# port:
# secure:
# accessKey:
# secretKey:
#
# Below settings are optional
#

View file

@ -9,7 +9,7 @@ const q = {
'metadata._user.host': {
$ne: null
},
'metadata.isMetaOnly': false
'metadata.withoutChunks': false
};
async function main() {
@ -57,7 +57,7 @@ async function main() {
DriveFile.update({ _id: file._id }, {
$set: {
'metadata.isMetaOnly': true
'metadata.withoutChunks': true
}
})
]).then(async () => {

View file

@ -3,8 +3,8 @@
const chalk = require('chalk');
const sequential = require('promise-sequential');
const { default: User } = require('../built/models/user');
const { default: DriveFile } = require('../built/models/drive-file');
const { default: User } = require('../../built/models/user');
const { default: DriveFile } = require('../../built/models/drive-file');
async function main() {
const promiseGens = [];

View file

@ -3,8 +3,8 @@
const chalk = require('chalk');
const sequential = require('promise-sequential');
const { default: User } = require('../built/models/user');
const { default: DriveFile } = require('../built/models/drive-file');
const { default: User } = require('../../built/models/user');
const { default: DriveFile } = require('../../built/models/drive-file');
async function main() {
const promiseGens = [];

10
cli/migration/5.0.0.js Normal file
View file

@ -0,0 +1,10 @@
const { default: DriveFile } = require('../../built/models/drive-file');
DriveFile.update({}, {
$rename: {
'metadata.url': 'metadata.src',
'metadata.isMetaOnly': 'metadata.withoutChunks',
}
}, {
multi: true
});

View file

@ -1,11 +0,0 @@
Misskeyの破壊的変更に対応するいくつかのスニペットがあります。
MongoDBシェルで実行する必要のあるものとnodeで直接実行する必要のあるものがあります。
ファイル名が `shell.` から始まるものは前者、 `node.` から始まるものは後者です。
MongoDBシェルで実行する場合、`use`でデータベースを選択しておく必要があります。
nodeで実行するいくつかのスニペットは、並列処理させる数を引数で設定できるものがあります。
処理中にエラーで落ちる場合は、メモリが足りていない可能性があるので、少ない数に設定してみてください。
※デフォルトは`5`です。
ファイルを作成する際は `../init-migration-file.sh -t _type_ -n _name_` を実行すると _type_._unixtime_._name_.js が生成されます

View file

@ -1,37 +0,0 @@
#!/bin/bash
usage() {
echo "$0 [-t type] [-n name]"
echo " type: [node | shell]"
echo " name: if no present, set untitled"
exit 0
}
while getopts :t:n:h OPT
do
case $OPT in
t) type=$OPTARG
;;
n) name=$OPTARG
;;
h) usage
;;
\?) usage
;;
:) usage
;;
esac
done
if [ "$type" = "" ]
then
echo "no type present!!!"
usage
fi
if [ "$name" = "" ]
then
name="untitled"
fi
touch "$(realpath $(dirname $BASH_SOURCE))/migration/$type.$(date +%s).$name.js"

View file

@ -1,7 +1,7 @@
{
"name": "misskey",
"author": "syuilo <i@syuilo.com>",
"version": "4.27.0",
"version": "5.0.0",
"clientVersion": "1.0.7487",
"codename": "nighthike",
"main": "./built/index.js",
@ -57,6 +57,7 @@
"@types/koa-views": "2.0.3",
"@types/koa__cors": "2.2.2",
"@types/kue": "0.11.9",
"@types/minio": "6.0.2",
"@types/mkdirp": "0.5.2",
"@types/mocha": "5.2.3",
"@types/mongodb": "3.1.2",
@ -147,6 +148,7 @@
"kue": "0.11.6",
"loader-utils": "1.1.0",
"mecab-async": "0.1.2",
"minio": "6.0.0",
"mkdirp": "0.5.1",
"mocha": "5.2.0",
"moji": "0.5.1",

View file

@ -2,7 +2,7 @@
<div class="form">
<header>
<h1><i>{{ app.name }}</i>があなたのアカウントにアクセスすることを<b>許可</b>しますか</h1>
<img :src="`${app.iconUrl}?thumbnail&size=64`"/>
<img :src="app.iconUrl"/>
</header>
<div class="app">
<section>

View file

@ -17,21 +17,21 @@ export default function(type, data): Notification {
return {
title: 'ファイルがアップロードされました',
body: data.name,
icon: data.url + '?thumbnail&size=64'
icon: data.url
};
case 'unread_messaging_message':
return {
title: `${getUserName(data.user)}さんからメッセージ:`,
body: data.text, // TODO: getMessagingMessageSummary(data),
icon: data.user.avatarUrl + '?thumbnail&size=64'
icon: data.user.avatarUrl
};
case 'reversi_invited':
return {
title: '対局への招待があります',
body: `${getUserName(data.parent)}さんから`,
icon: data.parent.avatarUrl + '?thumbnail&size=64'
icon: data.parent.avatarUrl
};
case 'notification':
@ -40,28 +40,28 @@ export default function(type, data): Notification {
return {
title: `${getUserName(data.user)}さんから:`,
body: getNoteSummary(data),
icon: data.user.avatarUrl + '?thumbnail&size=64'
icon: data.user.avatarUrl
};
case 'reply':
return {
title: `${getUserName(data.user)}さんから返信:`,
body: getNoteSummary(data),
icon: data.user.avatarUrl + '?thumbnail&size=64'
icon: data.user.avatarUrl
};
case 'quote':
return {
title: `${getUserName(data.user)}さんが引用:`,
body: getNoteSummary(data),
icon: data.user.avatarUrl + '?thumbnail&size=64'
icon: data.user.avatarUrl
};
case 'reaction':
return {
title: `${getUserName(data.user)}: ${getReactionEmoji(data.reaction)}:`,
body: getNoteSummary(data.note),
icon: data.user.avatarUrl + '?thumbnail&size=64'
icon: data.user.avatarUrl
};
default:

View file

@ -2,7 +2,7 @@
<div class="mk-autocomplete" @contextmenu.prevent="() => {}">
<ol class="users" ref="suggests" v-if="users.length > 0">
<li v-for="user in users" @click="complete(type, user)" @keydown="onKeydown" tabindex="-1">
<img class="avatar" :src="`${user.avatarUrl}?thumbnail&size=32`" alt=""/>
<img class="avatar" :src="user.avatarUrl" alt=""/>
<span class="name">{{ user | userName }}</span>
<span class="username">@{{ user | acct }}</span>
</li>

View file

@ -31,7 +31,7 @@ export default Vue.extend({
: this.user.avatarColor && this.user.avatarColor.length == 3
? `rgb(${ this.user.avatarColor.join(',') })`
: null,
backgroundImage: this.lightmode ? null : `url(${ this.user.avatarUrl }?thumbnail)`,
backgroundImage: this.lightmode ? null : `url(${ this.user.avatarUrl })`,
borderRadius: this.$store.state.settings.circleIcons ? '100%' : null
};
}

View file

@ -26,8 +26,8 @@
:class="{ empty: stone == null, none: o.map[i] == 'null', isEnded: game.isEnded, myTurn: !game.isEnded && isMyTurn, can: turnUser ? o.canPut(turnUser.id == blackUser.id, i) : null, prev: o.prevPos == i }"
@click="set(i)"
:title="`${String.fromCharCode(65 + o.transformPosToXy(i)[0])}${o.transformPosToXy(i)[1] + 1}`">
<img v-if="stone === true" :src="`${blackUser.avatarUrl}?thumbnail&size=128`" alt="">
<img v-if="stone === false" :src="`${whiteUser.avatarUrl}?thumbnail&size=128`" alt="">
<img v-if="stone === true" :src="blackUser.avatarUrl" alt="">
<img v-if="stone === false" :src="whiteUser.avatarUrl" alt="">
</div>
</div>
<div class="labels-y" v-if="this.$store.state.settings.reversiBoardLabels">

View file

@ -5,7 +5,7 @@
<p :class="$style.fetching" v-if="fetching">%fa:spinner .pulse .fw%%i18n:common.loading%<mk-ellipsis/></p>
<div :class="$style.stream" v-if="!fetching && images.length > 0">
<div v-for="image in images" :class="$style.img" :style="`background-image: url(${image.url}?thumbnail&size=256)`"></div>
<div v-for="image in images" :class="$style.img" :style="`background-image: url(${image.url})`"></div>
</div>
<p :class="$style.empty" v-if="!fetching && images.length == 0">%i18n:@no-photos%</p>
</mk-widget-container>

View file

@ -72,7 +72,7 @@ export default define({
if (this.images.length == 0) return;
const index = Math.floor(Math.random() * this.images.length);
const img = `url(${ this.images[index].url }?thumbnail&size=1024)`;
const img = `url(${ this.images[index].url })`;
(this.$refs.slideB as any).style.backgroundImage = img;

View file

@ -16,7 +16,7 @@
<p>%i18n:@banner%</p>
</div>
<div class="thumbnail" ref="thumbnail" :style="`background-color: ${ background }`">
<img :src="`${file.url}?thumbnail&size=128`" alt="" @load="onThumbnailLoaded"/>
<img :src="file.url" alt="" @load="onThumbnailLoaded"/>
</div>
<p class="name">
<span>{{ file.name.lastIndexOf('.') != -1 ? file.name.substr(0, file.name.lastIndexOf('.')) : file.name }}</span>

View file

@ -1,7 +1,7 @@
<template>
<mk-window width="400px" height="550px" @closed="$destroy">
<span slot="header" :class="$style.header">
<img :src="`${user.avatarUrl}?thumbnail&size=64`" alt=""/>{{ '%i18n:@followers%'.replace('{}', name) }}
<img :src="user.avatarUrl" alt=""/>{{ '%i18n:@followers%'.replace('{}', name) }}
</span>
<mk-followers :user="user"/>
</mk-window>

View file

@ -1,7 +1,7 @@
<template>
<mk-window width="400px" height="550px" @closed="$destroy">
<span slot="header" :class="$style.header">
<img :src="`${user.avatarUrl}?thumbnail&size=64`" alt=""/>{{ '%i18n:@following%'.replace('{}', name) }}
<img :src="user.avatarUrl" alt=""/>{{ '%i18n:@following%'.replace('{}', name) }}
</span>
<mk-following :user="user"/>
</mk-window>

View file

@ -37,7 +37,7 @@ export default Vue.extend({
style(): any {
return {
'background-color': this.image.properties.avgColor && this.image.properties.avgColor.length == 3 ? `rgb(${this.image.properties.avgColor.join(',')})` : 'transparent',
'background-image': this.raw ? `url(${this.image.url})` : `url(${this.image.url}?thumbnail&size=512)`
'background-image': this.raw ? `url(${this.image.url})` : `url(${this.image.url})`
};
}
},

View file

@ -45,7 +45,7 @@ export default Vue.extend({
computed: {
imageStyle(): any {
return {
'background-image': `url(${this.video.url}?thumbnail&size=512)`
'background-image': `url(${this.video.url})`
};
}
},

View file

@ -23,7 +23,7 @@
<div class="medias" :class="{ with: poll }" v-show="files.length != 0">
<x-draggable :list="files" :options="{ animation: 150 }">
<div v-for="file in files" :key="file.id">
<div class="img" :style="{ backgroundImage: `url(${file.url}?thumbnail&size=64)` }" :title="file.name"></div>
<div class="img" :style="{ backgroundImage: `url(${file.url})` }" :title="file.name"></div>
<img class="remove" @click="detachMedia(file.id)" src="/assets/desktop/remove.png" title="%i18n:@attach-cancel%" alt=""/>
</div>
</x-draggable>

View file

@ -2,7 +2,7 @@
<div class="profile">
<label class="avatar ui from group">
<p>%i18n:@avatar%</p>
<img class="avatar" :src="`${$store.state.i.avatarUrl}?thumbnail&size=64`" alt="avatar"/>
<img class="avatar" :src="$store.state.i.avatarUrl" alt="avatar"/>
<button class="ui" @click="updateAvatar">%i18n:@choice-avatar%</button>
</label>
<label class="ui from group">

View file

@ -1,7 +1,7 @@
<template>
<div class="mk-user-preview">
<template v-if="u != null">
<div class="banner" :style="u.bannerUrl ? `background-image: url(${u.bannerUrl}?thumbnail&size=512)` : ''"></div>
<div class="banner" :style="u.bannerUrl ? `background-image: url(${u.bannerUrl})` : ''"></div>
<mk-avatar class="avatar" :user="u" :disable-preview="true"/>
<div class="title">
<router-link class="name" :to="u | userPage">{{ u | userName }}</router-link>

View file

@ -4,7 +4,7 @@
<p class="initializing" v-if="fetching">%fa:spinner .pulse .fw%%i18n:@loading%<mk-ellipsis/></p>
<div v-if="!fetching && users.length > 0">
<router-link v-for="user in users" :to="user | userPage" :key="user.id">
<img :src="`${user.avatarUrl}?thumbnail&size=64`" :alt="user | userName" v-user-preview="user.id"/>
<img :src="user.avatarUrl" :alt="user | userName" v-user-preview="user.id"/>
</router-link>
</div>
<p class="empty" v-if="!fetching && users.length == 0">%i18n:@no-users%</p>

View file

@ -4,7 +4,7 @@
<p class="initializing" v-if="fetching">%fa:spinner .pulse .fw%%i18n:@loading%<mk-ellipsis/></p>
<div class="stream" v-if="!fetching && images.length > 0">
<div v-for="image in images" class="img"
:style="`background-image: url(${image.url}?thumbnail&size=256)`"
:style="`background-image: url(${image.url})`"
></div>
</div>
<p class="empty" v-if="!fetching && images.length == 0">%i18n:@no-photos%</p>

View file

@ -4,7 +4,7 @@
:data-melt="props.design == 2"
>
<div class="banner"
:style="$store.state.i.bannerUrl ? `background-image: url(${$store.state.i.bannerUrl}?thumbnail&size=256)` : ''"
:style="$store.state.i.bannerUrl ? `background-image: url(${$store.state.i.bannerUrl})` : ''"
title="%i18n:@update-banner%"
@click="os.apis.updateBanner"
></div>

View file

@ -43,7 +43,7 @@ export default Vue.extend({
thumbnail(): any {
return {
'background-color': this.file.properties.avgColor && this.file.properties.avgColor.length == 3 ? `rgb(${this.file.properties.avgColor.join(',')})` : 'transparent',
'background-image': `url(${this.file.url}?thumbnail&size=128)`
'background-image': `url(${this.file.url})`
};
}
},

View file

@ -27,7 +27,7 @@ export default Vue.extend({
},
computed: {
style(): any {
let url = `url(${this.image.url}?thumbnail)`;
let url = `url(${this.image.url})`;
if (this.$store.state.device.loadRemoteMedia || this.$store.state.device.lightmode) {
url = null;

View file

@ -30,7 +30,7 @@ export default Vue.extend({
computed: {
imageStyle(): any {
return {
'background-image': `url(${this.video.url}?thumbnail&size=512)`
'background-image': `url(${this.video.url})`
};
}
},})

View file

@ -2,7 +2,7 @@
<div class="mk-note-card">
<a :href="note | notePage">
<header>
<img :src="`${note.user.avatarUrl}?thumbnail&size=64`" alt="avatar"/><h3>{{ note.user | userName }}</h3>
<img :src="note.user.avatarUrl" alt="avatar"/><h3>{{ note.user | userName }}</h3>
</header>
<div>
{{ text }}

View file

@ -21,7 +21,7 @@
<div class="attaches" v-show="files.length != 0">
<x-draggable class="files" :list="files" :options="{ animation: 150 }">
<div class="file" v-for="file in files" :key="file.id">
<div class="img" :style="`background-image: url(${file.url}?thumbnail&size=128)`" @click="detachMedia(file)"></div>
<div class="img" :style="`background-image: url(${file.url})`" @click="detachMedia(file)"></div>
</div>
</x-draggable>
</div>

View file

@ -10,7 +10,7 @@
<transition name="nav">
<div class="body" v-if="isOpen">
<router-link class="me" v-if="$store.getters.isSignedIn" :to="`/@${$store.state.i.username}`">
<img class="avatar" :src="`${$store.state.i.avatarUrl}?thumbnail&size=128`" alt="avatar"/>
<img class="avatar" :src="$store.state.i.avatarUrl" alt="avatar"/>
<p class="name">{{ $store.state.i | userName }}</p>
</router-link>
<div class="links">

View file

@ -1,6 +1,6 @@
<template>
<div class="mk-user-card">
<header :style="user.bannerUrl ? `background-image: url(${user.bannerUrl}?thumbnail&size=1024)` : ''">
<header :style="user.bannerUrl ? `background-image: url(${user.bannerUrl})` : ''">
<mk-avatar class="avatar" :user="user"/>
</header>
<a class="name" :href="user | userPage" target="_blank">{{ user | userName }}</a>

View file

@ -1,7 +1,7 @@
<template>
<mk-ui>
<template slot="header" v-if="!fetching">
<img :src="`${user.avatarUrl}?thumbnail&size=64`" alt="">
<img :src="user.avatarUrl" alt="">
{{ '%i18n:@followers-of%'.replace('{}', name) }}
</template>
<mk-users-list

View file

@ -1,7 +1,7 @@
<template>
<mk-ui>
<template slot="header" v-if="!fetching">
<img :src="`${user.avatarUrl}?thumbnail&size=64`" alt="">
<img :src="user.avatarUrl" alt="">
{{ '%i18n:@following-of%'.replace('{}', name) }}
</template>
<mk-users-list

View file

@ -1,6 +1,6 @@
<template>
<mk-ui>
<template slot="header" v-if="!fetching"><img :src="`${user.avatarUrl}?thumbnail&size=64`" alt="">{{ user | userName }}</template>
<template slot="header" v-if="!fetching"><img :src="user.avatarUrl" alt="">{{ user | userName }}</template>
<main v-if="!fetching" :data-darkmode="$store.state.device.darkmode">
<div class="is-suspended" v-if="user.isSuspended"><p>%fa:exclamation-triangle% %i18n:@is-suspended%</p></div>
<div class="is-remote" v-if="user.host != null"><p>%fa:exclamation-triangle% %i18n:@is-remote%<a :href="user.url || user.uri" target="_blank">%i18n:@view-remote%</a></p></div>

View file

@ -3,7 +3,7 @@
<p class="initializing" v-if="fetching">%fa:spinner .pulse .fw%%i18n:@loading%<mk-ellipsis/></p>
<div v-if="!fetching && users.length > 0">
<a v-for="user in users" :key="user.id" :href="user | userPage">
<img :src="`${user.avatarUrl}?thumbnail&size=64`" :alt="user | userName"/>
<img :src="user.avatarUrl" :alt="user | userName"/>
</a>
</div>
<p class="empty" v-if="!fetching && users.length == 0">%i18n:@no-users%</p>

View file

@ -4,7 +4,7 @@
<div class="stream" v-if="!fetching && images.length > 0">
<a v-for="image in images"
class="img"
:style="`background-image: url(${image.media.url}?thumbnail&size=256)`"
:style="`background-image: url(${image.media.url})`"
:href="image.note | notePage"
></a>
</div>

View file

@ -2,10 +2,10 @@
<div class="mkw-profile">
<mk-widget-container>
<div :class="$style.banner"
:style="$store.state.i.bannerUrl ? `background-image: url(${$store.state.i.bannerUrl}?thumbnail&size=256)` : ''"
:style="$store.state.i.bannerUrl ? `background-image: url(${$store.state.i.bannerUrl})` : ''"
></div>
<img :class="$style.avatar"
:src="`${$store.state.i.avatarUrl}?thumbnail&size=96`"
:src="$store.state.i.avatarUrl"
alt="avatar"
/>
<router-link :class="$style.name" :to="$store.state.i | userPage">{{ $store.state.i | userName }}</router-link>

View file

@ -49,6 +49,14 @@ export type Source = {
remoteDriveCapacityMb: number;
preventCacheRemoteFiles: boolean;
drive?: {
storage: string;
bucket: string;
prefix: string;
service?: string;
config?: any;
};
/**
* ID
*/

View file

@ -31,8 +31,11 @@ export type IMetadata = {
comment: string;
uri?: string;
url?: string;
src?: string;
deletedAt?: Date;
isMetaOnly?: boolean;
withoutChunks?: boolean;
storage?: string;
storageProps?: any;
isSensitive?: boolean;
};
@ -155,9 +158,9 @@ export const pack = (
_target = Object.assign(_target, _file.metadata);
_target.url = _file.metadata.url ? _file.metadata.url : `${config.drive_url}/${_target.id}/${encodeURIComponent(_target.name)}`;
_target.src = _file.metadata.url;
_target.url = _file.metadata.isMetaOnly ? _file.metadata.url : `${config.drive_url}/${_target.id}/${encodeURIComponent(_target.name)}`;
_target.isRemote = _file.metadata.isMetaOnly;
_target.isRemote = _file.metadata.withoutChunks;
if (_target.properties == null) _target.properties = {};

View file

@ -152,8 +152,8 @@ export async function createPerson(value: any, resolver?: Resolver): Promise<IUs
const avatarId = avatar ? avatar._id : null;
const bannerId = banner ? banner._id : null;
const avatarUrl = avatar && avatar.metadata.isMetaOnly ? avatar.metadata.url : null;
const bannerUrl = banner && banner.metadata.isMetaOnly ? banner.metadata.url : null;
const avatarUrl = avatar && avatar.metadata.url ? avatar.metadata.url : null;
const bannerUrl = banner && banner.metadata.url ? banner.metadata.url : null;
await User.update({ _id: user._id }, {
$set: {
@ -243,8 +243,8 @@ export async function updatePerson(value: string | IObject, resolver?: Resolver)
sharedInbox: person.sharedInbox,
avatarId: avatar ? avatar._id : null,
bannerId: banner ? banner._id : null,
avatarUrl: avatar && avatar.metadata.isMetaOnly ? avatar.metadata.url : null,
bannerUrl: banner && banner.metadata.isMetaOnly ? banner.metadata.url : null,
avatarUrl: avatar && avatar.metadata.url ? avatar.metadata.url : null,
bannerUrl: banner && banner.metadata.url ? banner.metadata.url : null,
description: htmlToMFM(person.summary),
followersCount,
followingCount,

View file

@ -37,7 +37,7 @@ export default async function(ctx: Koa.Context) {
return;
}
if (file.metadata.isMetaOnly) {
if (file.metadata.withoutChunks) {
ctx.status = 204;
return;
}

View file

@ -8,14 +8,14 @@ import * as _gm from 'gm';
import * as debug from 'debug';
import fileType = require('file-type');
const prominence = require('prominence');
import * as Minio from 'minio';
import * as uuid from 'uuid';
import DriveFile, { IMetadata, getDriveFileBucket, IDriveFile } from '../../models/drive-file';
import DriveFolder from '../../models/drive-folder';
import { pack } from '../../models/drive-file';
import event, { publishDriveStream } from '../../stream';
import { isLocalUser, IUser, IRemoteUser } from '../../models/user';
import { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail';
import genThumbnail from '../../drive/gen-thumbnail';
import delFile from './delete-file';
import config from '../../config';
@ -25,28 +25,47 @@ const gm = _gm.subClass({
const log = debug('misskey:drive:add-file');
const writeChunks = (name: string, readable: stream.Readable, type: string, metadata: any) =>
getDriveFileBucket()
.then(bucket => new Promise((resolve, reject) => {
async function save(readable: stream.Readable, name: string, type: string, hash: string, size: number, metadata: any): Promise<IDriveFile> {
if (config.drive && config.drive.storage == 'object-storage') {
if (config.drive.service == 'minio') {
const minio = new Minio.Client(config.drive.config);
const id = uuid.v4();
const obj = `${config.drive.prefix}/${id}`;
await minio.putObject(config.drive.bucket, obj, readable);
Object.assign(metadata, {
withoutChunks: true,
storage: 'object-storage',
storageProps: {
id: id
},
url: `${ config.drive.config.secure ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? ':' + config.drive.config.port : '' }/${ config.drive.bucket }/${ obj }`
});
const file = await DriveFile.insert({
length: size,
uploadDate: new Date(),
md5: hash,
filename: name,
metadata: metadata,
contentType: type
});
return file;
}
} else {
// Get MongoDB GridFS bucket
const bucket = await getDriveFileBucket();
return new Promise<IDriveFile>((resolve, reject) => {
const writeStream = bucket.openUploadStream(name, { contentType: type, metadata });
writeStream.once('finish', resolve);
writeStream.on('error', reject);
readable.pipe(writeStream);
}));
const writeThumbnailChunks = (name: string, readable: stream.Readable, originalId: mongodb.ObjectID) =>
getDriveFileThumbnailBucket()
.then(bucket => new Promise((resolve, reject) => {
const writeStream = bucket.openUploadStream(name, {
contentType: 'image/jpeg',
metadata: {
originalId
}
});
writeStream.once('finish', resolve);
writeStream.on('error', reject);
readable.pipe(writeStream);
}));
});
}
}
async function deleteOldFile(user: IRemoteUser) {
const oldFile = await DriveFile.findOne({
@ -82,7 +101,7 @@ export default async function(
comment: string = null,
folderId: mongodb.ObjectID = null,
force: boolean = false,
metaOnly: boolean = false,
isLink: boolean = false,
url: string = null,
uri: string = null,
sensitive = false
@ -150,7 +169,7 @@ export default async function(
}
//#region Check drive usage
if (!metaOnly) {
if (!isLink) {
const usage = await DriveFile
.aggregate([{
$match: {
@ -262,19 +281,23 @@ export default async function(
folderId: folder !== null ? folder._id : null,
comment: comment,
properties: properties,
isMetaOnly: metaOnly,
withoutChunks: isLink,
isSensitive: sensitive
} as IMetadata;
if (url !== null) {
metadata.url = url;
metadata.src = url;
if (isLink) {
metadata.url = url;
}
}
if (uri !== null) {
metadata.uri = uri;
}
const driveFile = metaOnly
const driveFile = isLink
? await DriveFile.insert({
length: 0,
uploadDate: new Date(),
@ -283,7 +306,7 @@ export default async function(
metadata: metadata,
contentType: mime
})
: await (writeChunks(detectedName, fs.createReadStream(path), mime, metadata) as Promise<IDriveFile>);
: await (save(fs.createReadStream(path), detectedName, mime, hash, size, metadata));
log(`drive file has been created ${driveFile._id}`);
@ -293,16 +316,7 @@ export default async function(
publishDriveStream(user._id, 'file_created', packedFile);
});
if (!metaOnly) {
try {
const thumb = await genThumbnail(driveFile);
if (thumb) {
await writeThumbnailChunks(detectedName, thumb, driveFile._id);
}
} catch (e) {
// noop
}
}
// TODO: サムネイル生成
return driveFile;
}

View file

@ -1,30 +1,40 @@
import * as Minio from 'minio';
import DriveFile, { DriveFileChunk, IDriveFile } from '../../models/drive-file';
import DriveFileThumbnail, { DriveFileThumbnailChunk } from '../../models/drive-file-thumbnail';
import config from '../../config';
export default async function(file: IDriveFile, isExpired = false) {
// チャンクをすべて削除
await DriveFileChunk.remove({
files_id: file._id
});
await DriveFile.update({ _id: file._id }, {
$set: {
'metadata.deletedAt': new Date(),
'metadata.isExpired': isExpired
if (file.metadata.withoutChunks) {
if (file.metadata.storage == 'object-storage') {
const minio = new Minio.Client(config.drive.config);
const obj = `${config.drive.prefix}/${file.metadata.storageProps.id}`;
await minio.removeObject(config.drive.bucket, obj);
}
});
//#region サムネイルもあれば削除
const thumbnail = await DriveFileThumbnail.findOne({
'metadata.originalId': file._id
});
if (thumbnail) {
await DriveFileThumbnailChunk.remove({
files_id: thumbnail._id
} else {
// チャンクをすべて削除
await DriveFileChunk.remove({
files_id: file._id
});
await DriveFileThumbnail.remove({ _id: thumbnail._id });
await DriveFile.update({ _id: file._id }, {
$set: {
'metadata.deletedAt': new Date(),
'metadata.isExpired': isExpired
}
});
//#region サムネイルもあれば削除
const thumbnail = await DriveFileThumbnail.findOne({
'metadata.originalId': file._id
});
if (thumbnail) {
await DriveFileThumbnailChunk.remove({
files_id: thumbnail._id
});
await DriveFileThumbnail.remove({ _id: thumbnail._id });
}
//#endregion
}
//#endregion
}