From 0c9a52940f59d3abb039104609e92ee327427e74 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sun, 19 Jul 2020 00:24:07 +0900 Subject: [PATCH] feat: Blurhash integration Resolve #6559 --- migration/1595075960584-blurhash.ts | 14 +++ ...595077605646-blurhash-for-avatar-banner.ts | 20 ++++ package.json | 1 + src/client/components/avatar.vue | 38 ++------ .../components/drive-file-thumbnail.vue | 85 +++-------------- src/client/components/drive.file.vue | 12 --- src/client/components/img-with-blurhash.vue | 78 ++++++++++++++++ src/client/components/media-image.vue | 91 +++++++++++-------- src/client/components/media-list.vue | 2 +- .../messaging/messaging-room.message.vue | 3 +- .../page-editor/els/page-editor.el.image.vue | 2 +- src/client/style.scss | 4 - src/misc/get-file-info.ts | 37 ++++---- src/models/entities/drive-file.ts | 6 ++ src/models/entities/user.ts | 8 +- src/models/repositories/drive-file.ts | 1 + src/models/repositories/user.ts | 10 +- src/remote/activitypub/models/person.ts | 16 ++-- src/server/api/endpoints/i/update.ts | 8 +- src/services/drive/add-file.ts | 6 +- test/get-file-info.ts | 16 ++-- yarn.lock | 5 + 22 files changed, 249 insertions(+), 214 deletions(-) create mode 100644 migration/1595075960584-blurhash.ts create mode 100644 migration/1595077605646-blurhash-for-avatar-banner.ts create mode 100644 src/client/components/img-with-blurhash.vue diff --git a/migration/1595075960584-blurhash.ts b/migration/1595075960584-blurhash.ts new file mode 100644 index 000000000..7c716ae17 --- /dev/null +++ b/migration/1595075960584-blurhash.ts @@ -0,0 +1,14 @@ +import {MigrationInterface, QueryRunner} from "typeorm"; + +export class blurhash1595075960584 implements MigrationInterface { + name = 'blurhash1595075960584' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "drive_file" ADD "blurhash" character varying(128)`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "drive_file" DROP COLUMN "blurhash"`); + } + +} diff --git a/migration/1595077605646-blurhash-for-avatar-banner.ts b/migration/1595077605646-blurhash-for-avatar-banner.ts new file mode 100644 index 000000000..fcf161c35 --- /dev/null +++ b/migration/1595077605646-blurhash-for-avatar-banner.ts @@ -0,0 +1,20 @@ +import {MigrationInterface, QueryRunner} from "typeorm"; + +export class blurhashForAvatarBanner1595077605646 implements MigrationInterface { + name = 'blurhashForAvatarBanner1595077605646' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "user" DROP COLUMN "avatarColor"`); + await queryRunner.query(`ALTER TABLE "user" DROP COLUMN "bannerColor"`); + await queryRunner.query(`ALTER TABLE "user" ADD "avatarBlurhash" character varying(128)`); + await queryRunner.query(`ALTER TABLE "user" ADD "bannerBlurhash" character varying(128)`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "user" DROP COLUMN "bannerBlurhash"`); + await queryRunner.query(`ALTER TABLE "user" DROP COLUMN "avatarBlurhash"`); + await queryRunner.query(`ALTER TABLE "user" ADD "bannerColor" character varying(32)`); + await queryRunner.query(`ALTER TABLE "user" ADD "avatarColor" character varying(32)`); + } + +} diff --git a/package.json b/package.json index 8868fede5..eef08c93a 100644 --- a/package.json +++ b/package.json @@ -112,6 +112,7 @@ "autwh": "0.1.0", "aws-sdk": "2.713.0", "bcryptjs": "2.4.3", + "blurhash": "1.1.3", "bull": "3.15.0", "cafy": "15.2.1", "cbor": "5.0.2", diff --git a/src/client/components/avatar.vue b/src/client/components/avatar.vue index 29b457db8..fd4ab78ce 100644 --- a/src/client/components/avatar.vue +++ b/src/client/components/avatar.vue @@ -1,15 +1,9 @@ @@ -45,22 +39,6 @@ export default Vue.extend({ ? getStaticImageUrl(this.user.avatarUrl) : this.user.avatarUrl; }, - icon(): any { - return { - backgroundColor: this.user.avatarColor, - backgroundImage: `url(${this.url})`, - }; - } - }, - watch: { - 'user.avatarColor'() { - this.$el.style.color = this.user.avatarColor; - } - }, - mounted() { - if (this.user.avatarColor) { - this.$el.style.color = this.user.avatarColor; - } }, methods: { onClick(e) { @@ -102,15 +80,17 @@ export default Vue.extend({ } .inner { - background-position: center center; - background-size: cover; + position: absolute; bottom: 0; left: 0; - position: absolute; right: 0; top: 0; border-radius: 100%; z-index: 1; + overflow: hidden; + object-fit: cover; + width: 100%; + height: 100%; } } diff --git a/src/client/components/drive-file-thumbnail.vue b/src/client/components/drive-file-thumbnail.vue index 3561be0bc..4bc1e569b 100644 --- a/src/client/components/drive-file-thumbnail.vue +++ b/src/client/components/drive-file-thumbnail.vue @@ -1,36 +1,15 @@ @@ -47,8 +26,12 @@ import { faFileArchive, faFilm } from '@fortawesome/free-solid-svg-icons'; +import ImgWithBlurhash from './img-with-blurhash.vue'; export default Vue.extend({ + components: { + ImgWithBlurhash + }, props: { file: { type: Object, @@ -59,11 +42,6 @@ export default Vue.extend({ required: false, default: 'cover' }, - detail: { - type: Boolean, - required: false, - default: false - } }, data() { return { @@ -108,20 +86,12 @@ export default Vue.extend({ ? (this.is === 'image' || this.is === 'video') : false; }, - background(): string { - return this.file.properties.avgColor || 'transparent'; - } }, mounted() { const audioTag = this.$refs.volumectrl as HTMLAudioElement; if (audioTag) audioTag.volume = this.$store.state.device.mediaVolume; }, methods: { - onThumbnailLoaded() { - if (this.file.properties.avgColor) { - this.$refs.thumbnail.style.backgroundColor = 'transparent'; - } - }, volumechange() { const audioTag = this.$refs.volumectrl as HTMLAudioElement; this.$store.commit('device/set', { key: 'mediaVolume', value: audioTag.volume }); @@ -132,14 +102,8 @@ export default Vue.extend({ diff --git a/src/client/components/drive.file.vue b/src/client/components/drive.file.vue index 1b24c61df..b31a4e637 100644 --- a/src/client/components/drive.file.vue +++ b/src/client/components/drive.file.vue @@ -126,17 +126,6 @@ export default Vue.extend({ this.browser.isDragSource = false; }, - onThumbnailLoaded() { - if (this.file.properties.avgColor) { - anime({ - targets: this.$refs.thumbnail, - backgroundColor: 'transparent', // TODO fade - duration: 100, - easing: 'linear' - }); - } - }, - rename() { this.$root.dialog({ title: this.$t('renameFile'), @@ -332,7 +321,6 @@ export default Vue.extend({ width: 128px; height: 128px; margin: auto; - color: var(--driveFileIcon); } > .name { diff --git a/src/client/components/img-with-blurhash.vue b/src/client/components/img-with-blurhash.vue new file mode 100644 index 000000000..6e6a2a896 --- /dev/null +++ b/src/client/components/img-with-blurhash.vue @@ -0,0 +1,78 @@ + + + + + diff --git a/src/client/components/media-image.vue b/src/client/components/media-image.vue index 6d1b5345d..f6ed45dae 100644 --- a/src/client/components/media-image.vue +++ b/src/client/components/media-image.vue @@ -1,19 +1,22 @@ @@ -23,8 +26,12 @@ import Vue from 'vue'; import { faExclamationTriangle, faEyeSlash } from '@fortawesome/free-solid-svg-icons'; import { getStaticImageUrl } from '../scripts/get-static-image-url'; import ImageViewer from './image-viewer.vue'; +import ImgWithBlurhash from './img-with-blurhash.vue'; export default Vue.extend({ + components: { + ImgWithBlurhash + }, props: { image: { type: Object, @@ -42,23 +49,18 @@ export default Vue.extend({ }; }, computed: { - style(): any { - let url = `url(${ - this.$store.state.device.disableShowingAnimatedImages - ? getStaticImageUrl(this.image.thumbnailUrl) - : this.image.thumbnailUrl - })`; + url(): any { + let url = this.$store.state.device.disableShowingAnimatedImages + ? getStaticImageUrl(this.image.thumbnailUrl) + : this.image.thumbnailUrl; if (this.$store.state.device.loadRemoteMedia) { url = null; } else if (this.raw || this.$store.state.device.loadRawImages) { - url = `url(${this.image.url})`; + url = this.image.url; } - return { - 'background-color': this.image.properties.avgColor || 'transparent', - 'background-image': url - }; + return url; } }, created() { @@ -82,7 +84,38 @@ export default Vue.extend({ diff --git a/src/client/components/media-list.vue b/src/client/components/media-list.vue index c757d8091..fd0035f10 100644 --- a/src/client/components/media-list.vue +++ b/src/client/components/media-list.vue @@ -114,7 +114,7 @@ export default Vue.extend({ > * { overflow: hidden; - border-radius: 4px; + border-radius: 6px; } &[data-count="1"] { diff --git a/src/client/pages/messaging/messaging-room.message.vue b/src/client/pages/messaging/messaging-room.message.vue index 58e1e54ad..4461740df 100644 --- a/src/client/pages/messaging/messaging-room.message.vue +++ b/src/client/pages/messaging/messaging-room.message.vue @@ -10,8 +10,7 @@
- +

{{ message.file.name }}
diff --git a/src/client/pages/page-editor/els/page-editor.el.image.vue b/src/client/pages/page-editor/els/page-editor.el.image.vue index dd690da6f..d26d7f603 100644 --- a/src/client/pages/page-editor/els/page-editor.el.image.vue +++ b/src/client/pages/page-editor/els/page-editor.el.image.vue @@ -8,7 +8,7 @@
- +
diff --git a/src/client/style.scss b/src/client/style.scss index 972c38338..c3d3cf223 100644 --- a/src/client/style.scss +++ b/src/client/style.scss @@ -123,10 +123,6 @@ a { &:hover { text-decoration: underline; } - - * { - cursor: pointer; - } } hr { diff --git a/src/misc/get-file-info.ts b/src/misc/get-file-info.ts index b838900f6..ce177cc53 100644 --- a/src/misc/get-file-info.ts +++ b/src/misc/get-file-info.ts @@ -6,6 +6,7 @@ import * as fileType from 'file-type'; import isSvg from 'is-svg'; import * as probeImageSize from 'probe-image-size'; import * as sharp from 'sharp'; +import { encode } from 'blurhash'; const pipeline = util.promisify(stream.pipeline); @@ -18,7 +19,7 @@ export type FileInfo = { }; width?: number; height?: number; - avgColor?: number[]; + blurhash?: string; warnings: string[]; }; @@ -71,12 +72,11 @@ export async function getFileInfo(path: string): Promise { } } - // average color - let avgColor: number[] | undefined; + let blurhash: string | undefined; if (['image/jpeg', 'image/gif', 'image/png', 'image/apng', 'image/webp', 'image/svg+xml'].includes(type.mime)) { - avgColor = await calcAvgColor(path).catch(e => { - warnings.push(`calcAvgColor failed: ${e}`); + blurhash = await getBlurhash(path).catch(e => { + warnings.push(`getBlurhash failed: ${e}`); return undefined; }); } @@ -87,7 +87,7 @@ export async function getFileInfo(path: string): Promise { type, width, height, - avgColor, + blurhash, warnings, }; } @@ -173,18 +173,15 @@ async function detectImageSize(path: string): Promise<{ /** * Calculate average color of image */ -async function calcAvgColor(path: string): Promise { - const img = sharp(path); - - const info = await (img as any).stats(); - - if (info.isOpaque) { - const r = Math.round(info.channels[0].mean); - const g = Math.round(info.channels[1].mean); - const b = Math.round(info.channels[2].mean); - - return [r, g, b]; - } else { - return [255, 255, 255]; - } +function getBlurhash(path: string): Promise { + return new Promise((resolve, reject) => { + sharp(path) + .raw() + .ensureAlpha() + .resize(64, 64, { fit: 'inside' }) + .toBuffer((err, buffer, { width, height }) => { + if (err) return reject(err); + resolve(encode(new Uint8ClampedArray(buffer), width, height, 7, 7)); + }); + }); } diff --git a/src/models/entities/drive-file.ts b/src/models/entities/drive-file.ts index 067dc1181..c02b9f363 100644 --- a/src/models/entities/drive-file.ts +++ b/src/models/entities/drive-file.ts @@ -67,6 +67,12 @@ export class DriveFile { }) public comment: string | null; + @Column('varchar', { + length: 128, nullable: true, + comment: 'The BlurHash string.' + }) + public blurhash: string | null; + @Column('jsonb', { default: {}, comment: 'The any properties of the DriveFile. For example, it includes image width/height.' diff --git a/src/models/entities/user.ts b/src/models/entities/user.ts index d3086f43f..fee5906a3 100644 --- a/src/models/entities/user.ts +++ b/src/models/entities/user.ts @@ -106,14 +106,14 @@ export class User { public bannerUrl: string | null; @Column('varchar', { - length: 32, nullable: true, + length: 128, nullable: true, }) - public avatarColor: string | null; + public avatarBlurhash: string | null; @Column('varchar', { - length: 32, nullable: true, + length: 128, nullable: true, }) - public bannerColor: string | null; + public bannerBlurhash: string | null; @Column('boolean', { default: false, diff --git a/src/models/repositories/drive-file.ts b/src/models/repositories/drive-file.ts index 28a393cfb..6bdf62be8 100644 --- a/src/models/repositories/drive-file.ts +++ b/src/models/repositories/drive-file.ts @@ -115,6 +115,7 @@ export class DriveFileRepository extends Repository { md5: file.md5, size: file.size, isSensitive: file.isSensitive, + blurhash: file.blurhash, properties: file.properties, url: opts.self ? file.url : this.getPublicUrl(file, false, meta), thumbnailUrl: this.getPublicUrl(file, true, meta), diff --git a/src/models/repositories/user.ts b/src/models/repositories/user.ts index c4c9503e0..bbaafc905 100644 --- a/src/models/repositories/user.ts +++ b/src/models/repositories/user.ts @@ -165,7 +165,8 @@ export class UserRepository extends Repository { username: user.username, host: user.host, avatarUrl: user.avatarUrl ? user.avatarUrl : config.url + '/avatar/' + user.id, - avatarColor: user.avatarColor, + avatarBlurhash: user.avatarBlurhash, + avatarColor: null, // 後方互換性のため isAdmin: user.isAdmin || falsy, isModerator: user.isModerator || falsy, isBot: user.isBot || falsy, @@ -196,7 +197,8 @@ export class UserRepository extends Repository { createdAt: user.createdAt.toISOString(), updatedAt: user.updatedAt ? user.updatedAt.toISOString() : null, bannerUrl: user.bannerUrl, - bannerColor: user.bannerColor, + bannerBlurhash: user.bannerBlurhash, + bannerColor: null, // 後方互換性のため isLocked: user.isLocked, isModerator: user.isModerator || falsy, isSilenced: user.isSilenced || falsy, @@ -331,7 +333,7 @@ export const packedUserSchema = { format: 'url', nullable: true as const, optional: false as const, }, - avatarColor: { + avatarBlurhash: { type: 'any' as const, nullable: true as const, optional: false as const, }, @@ -340,7 +342,7 @@ export const packedUserSchema = { format: 'url', nullable: true as const, optional: true as const, }, - bannerColor: { + bannerBlurhash: { type: 'any' as const, nullable: true as const, optional: true as const, }, diff --git a/src/remote/activitypub/models/person.ts b/src/remote/activitypub/models/person.ts index a3093786d..a213abf47 100644 --- a/src/remote/activitypub/models/person.ts +++ b/src/remote/activitypub/models/person.ts @@ -226,24 +226,24 @@ export async function createPerson(uri: string, resolver?: Resolver): Promise { updates.avatarUrl = DriveFiles.getPublicUrl(avatar, true); - if (avatar.properties.avgColor) { - updates.avatarColor = avatar.properties.avgColor; + if (avatar.blurhash) { + updates.avatarBlurhash = avatar.blurhash; } } @@ -223,8 +223,8 @@ export default define(meta, async (ps, user, token) => { updates.bannerUrl = DriveFiles.getPublicUrl(banner, false); - if (banner.properties.avgColor) { - updates.bannerColor = banner.properties.avgColor; + if (banner.blurhash) { + updates.bannerBlurhash = banner.blurhash; } } diff --git a/src/services/drive/add-file.ts b/src/services/drive/add-file.ts index cf0951eba..969dc0406 100644 --- a/src/services/drive/add-file.ts +++ b/src/services/drive/add-file.ts @@ -327,7 +327,6 @@ export default async function( const properties: { width?: number; height?: number; - avgColor?: string; } = {}; if (info.width) { @@ -335,10 +334,6 @@ export default async function( properties['height'] = info.height; } - if (info.avgColor) { - properties['avgColor'] = `rgb(${info.avgColor.join(',')})`; - } - const profile = user ? await UserProfiles.findOne(user.id) : null; const folder = await fetchFolder(); @@ -351,6 +346,7 @@ export default async function( file.folderId = folder !== null ? folder.id : null; file.comment = comment; file.properties = properties; + file.blurhash = info.blurhash || null; file.isLink = isLink; file.isSensitive = user ? Users.isLocalUser(user) && profile!.alwaysMarkNsfw ? true : diff --git a/test/get-file-info.ts b/test/get-file-info.ts index 920df0738..0c19fb2d7 100644 --- a/test/get-file-info.ts +++ b/test/get-file-info.ts @@ -26,7 +26,7 @@ describe('Get file info', () => { }, width: undefined, height: undefined, - avgColor: undefined + blurhash: null }); })); @@ -43,7 +43,7 @@ describe('Get file info', () => { }, width: 512, height: 512, - avgColor: [ 181, 99, 106 ] + blurhash: '' // TODO }); })); @@ -60,7 +60,7 @@ describe('Get file info', () => { }, width: 256, height: 256, - avgColor: [ 249, 253, 250 ] + blurhash: '' // TODO }); })); @@ -77,7 +77,7 @@ describe('Get file info', () => { }, width: 256, height: 256, - avgColor: [ 249, 253, 250 ] + blurhash: '' // TODO }); })); @@ -94,7 +94,7 @@ describe('Get file info', () => { }, width: 256, height: 256, - avgColor: [ 255, 255, 255 ] + blurhash: '' // TODO }); })); @@ -111,7 +111,7 @@ describe('Get file info', () => { }, width: 256, height: 256, - avgColor: [ 255, 255, 255 ] + blurhash: '' // TODO }); })); @@ -129,7 +129,7 @@ describe('Get file info', () => { }, width: 256, height: 256, - avgColor: [ 255, 255, 255 ] + blurhash: '' // TODO }); })); @@ -146,7 +146,7 @@ describe('Get file info', () => { }, width: 25000, height: 25000, - avgColor: undefined + blurhash: '' // TODO }); })); }); diff --git a/yarn.lock b/yarn.lock index 2000b823e..55e62690a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1669,6 +1669,11 @@ bluebird@^3.1.1, bluebird@^3.4.1: resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== +blurhash@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/blurhash/-/blurhash-1.1.3.tgz#dc325af7da836d07a0861d830bdd63694382483e" + integrity sha512-yUhPJvXexbqbyijCIE/T2NCXcj9iNPhWmOKbPTuR/cm7Q5snXYIfnVnz6m7MWOXxODMz/Cr3UcVkRdHiuDVRDw== + bn.js@^4.0.0: version "4.11.8" resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.11.8.tgz#2cde09eb5ee341f484746bb0309b3253b1b1442f"