From 39c4c7835437fad0ae0bfa75d733916cbd44703e Mon Sep 17 00:00:00 2001 From: syuilo Date: Wed, 6 Sep 2017 23:19:58 +0900 Subject: [PATCH] wip --- locales/en.yml | 12 +++ locales/ja.yml | 12 +++ src/api/endpoints.ts | 4 + src/api/endpoints/posts/categorize.ts | 52 +++++++++++++ src/tools/ai/categorizer.ts | 93 ----------------------- src/tools/ai/predict-all-post-category.ts | 57 ++++++++++++++ src/tools/ai/predict-user-interst.ts | 45 +++++++++++ src/web/app/common/tags/post-menu.tag | 23 ++++++ 8 files changed, 205 insertions(+), 93 deletions(-) create mode 100644 src/api/endpoints/posts/categorize.ts delete mode 100644 src/tools/ai/categorizer.ts create mode 100644 src/tools/ai/predict-all-post-category.ts create mode 100644 src/tools/ai/predict-user-interst.ts diff --git a/locales/en.yml b/locales/en.yml index d40896212..3b87ea758 100644 --- a/locales/en.yml +++ b/locales/en.yml @@ -22,6 +22,14 @@ common: confused: "Confused" pudding: "Pudding" + post_categories: + music: "Music" + game: "Video Game" + anime: "Anime" + it: "IT" + gadgets: "Gadgets" + photography: "Photography" + input-message-here: "Enter message here" send: "Send" delete: "Delete" @@ -80,6 +88,9 @@ common: mk-post-menu: pin: "Pin" pinned: "Pinned" + select: "Select category" + categorize: "Accept" + categorized: "Category reported. Thank you!" mk-reaction-picker: choose-reaction: "Pick your reaction" @@ -375,6 +386,7 @@ mobile: twitter-integration: "Twitter integration" signin-history: "Sign in history" api: "API" + link: "MisskeyLink" settings: "Settings" signout: "Sign out" diff --git a/locales/ja.yml b/locales/ja.yml index b8e5cff41..13d451b6d 100644 --- a/locales/ja.yml +++ b/locales/ja.yml @@ -22,6 +22,14 @@ common: confused: "こまこまのこまり" pudding: "Pudding" + post_categories: + music: "音楽" + game: "ゲーム" + anime: "アニメ" + it: "IT" + gadgets: "ガジェット" + photography: "写真" + input-message-here: "ここにメッセージを入力" send: "送信" delete: "削除" @@ -80,6 +88,9 @@ common: mk-post-menu: pin: "ピン留め" pinned: "ピン留めしました" + select: "カテゴリを選択" + categorize: "決定" + categorized: "カテゴリを報告しました。これによりMisskeyが賢くなり、投稿の自動カテゴライズに役立てられます。ご協力ありがとうございました。" mk-reaction-picker: choose-reaction: "リアクションを選択" @@ -375,6 +386,7 @@ mobile: twitter-integration: "Twitter連携" signin-history: "ログイン履歴" api: "API" + link: "Misskeyリンク" settings: "設定" signout: "サインアウト" diff --git a/src/api/endpoints.ts b/src/api/endpoints.ts index e5be68c09..97b98895b 100644 --- a/src/api/endpoints.ts +++ b/src/api/endpoints.ts @@ -394,6 +394,10 @@ const endpoints: Endpoint[] = [ name: 'posts/trend', withCredential: true }, + { + name: 'posts/categorize', + withCredential: true + }, { name: 'posts/reactions', withCredential: true diff --git a/src/api/endpoints/posts/categorize.ts b/src/api/endpoints/posts/categorize.ts new file mode 100644 index 000000000..3530ba6bc --- /dev/null +++ b/src/api/endpoints/posts/categorize.ts @@ -0,0 +1,52 @@ +/** + * Module dependencies + */ +import $ from 'cafy'; +import Post from '../../models/post'; + +/** + * Categorize a post + * + * @param {any} params + * @param {any} user + * @return {Promise} + */ +module.exports = (params, user) => new Promise(async (res, rej) => { + if (!user.is_pro) { + return rej('This endpoint is available only from a Pro account'); + } + + // Get 'post_id' parameter + const [postId, postIdErr] = $(params.post_id).id().$; + if (postIdErr) return rej('invalid post_id param'); + + // Get categorizee + const post = await Post.findOne({ + _id: postId + }); + + if (post === null) { + return rej('post not found'); + } + + if (post.is_category_verified) { + return rej('This post already has the verified category'); + } + + // Get 'category' parameter + const [category, categoryErr] = $(params.category).string().or([ + 'music', 'game', 'anime', 'it', 'gadgets', 'photography' + ]).$; + if (categoryErr) return rej('invalid category param'); + + // Set category + Post.update({ _id: post._id }, { + $set: { + category: category, + is_category_verified: true + } + }); + + // Send response + res(); +}); diff --git a/src/tools/ai/categorizer.ts b/src/tools/ai/categorizer.ts deleted file mode 100644 index c13374161..000000000 --- a/src/tools/ai/categorizer.ts +++ /dev/null @@ -1,93 +0,0 @@ -import * as fs from 'fs'; - -const bayes = require('./naive-bayes.js'); -const MeCab = require('mecab-async'); -import * as msgpack from 'msgpack-lite'; - -import Post from '../../api/models/post'; -import config from '../../conf'; - -/** - * 投稿を学習したり与えられた投稿のカテゴリを予測します - */ -export default class Categorizer { - private classifier: any; - private categorizerDbFilePath: string; - private mecab: any; - - constructor() { - this.categorizerDbFilePath = `${__dirname}/../../../data/category`; - - this.mecab = new MeCab(); - if (config.categorizer.mecab_command) this.mecab.command = config.categorizer.mecab_command; - - // BIND ----------------------------------- - this.tokenizer = this.tokenizer.bind(this); - } - - private tokenizer(text: string) { - return this.mecab.wakachiSync(text); - } - - public async init() { - try { - const buffer = fs.readFileSync(this.categorizerDbFilePath); - const db = msgpack.decode(buffer); - - this.classifier = bayes.import(db); - this.classifier.tokenizer = this.tokenizer; - } catch (e) { - this.classifier = bayes({ - tokenizer: this.tokenizer - }); - - // 訓練データ - const verifiedPosts = await Post.find({ - is_category_verified: true - }); - - // 学習 - verifiedPosts.forEach(post => { - this.classifier.learn(post.text, post.category); - }); - - this.save(); - } - } - - public async learn(id, category) { - const post = await Post.findOne({ _id: id }); - - Post.update({ _id: id }, { - $set: { - category: category, - is_category_verified: true - } - }); - - this.classifier.learn(post.text, category); - - this.save(); - } - - public async categorize(id) { - const post = await Post.findOne({ _id: id }); - - const category = this.classifier.categorize(post.text); - - Post.update({ _id: id }, { - $set: { - category: category - } - }); - } - - public async test(text) { - return this.classifier.categorize(text); - } - - private save() { - const buffer = msgpack.encode(this.classifier.export()); - fs.writeFileSync(this.categorizerDbFilePath, buffer); - } -} diff --git a/src/tools/ai/predict-all-post-category.ts b/src/tools/ai/predict-all-post-category.ts new file mode 100644 index 000000000..87e198b39 --- /dev/null +++ b/src/tools/ai/predict-all-post-category.ts @@ -0,0 +1,57 @@ +const bayes = require('./naive-bayes.js'); +const MeCab = require('mecab-async'); + +import Post from '../../api/models/post'; +import config from '../../conf'; + +const classifier = bayes({ + tokenizer: this.tokenizer +}); + +const mecab = new MeCab(); +if (config.categorizer.mecab_command) mecab.command = config.categorizer.mecab_command; + +// 訓練データ取得 +Post.find({ + is_category_verified: true +}, { + fields: { + _id: false, + text: true, + category: true + } +}).then(verifiedPosts => { + // 学習 + verifiedPosts.forEach(post => { + classifier.learn(post.text, post.category); + }); + + // 全ての(人間によって証明されていない)投稿を取得 + Post.find({ + text: { + $exists: true + }, + is_category_verified: { + $ne: true + } + }, { + sort: { + _id: -1 + }, + fields: { + _id: true, + text: true + } + }).then(posts => { + posts.forEach(post => { + console.log(`predicting... ${post._id}`); + const category = classifier.categorize(post.text); + + Post.update({ _id: post._id }, { + $set: { + category: category + } + }); + }); + }); +}); diff --git a/src/tools/ai/predict-user-interst.ts b/src/tools/ai/predict-user-interst.ts new file mode 100644 index 000000000..99bdfa420 --- /dev/null +++ b/src/tools/ai/predict-user-interst.ts @@ -0,0 +1,45 @@ +import Post from '../../api/models/post'; +import User from '../../api/models/user'; + +export async function predictOne(id) { + console.log(`predict interest of ${id} ...`); + + // TODO: repostなども含める + const recentPosts = await Post.find({ + user_id: id, + category: { + $exists: true + } + }, { + sort: { + _id: -1 + }, + limit: 1000, + fields: { + _id: false, + category: true + } + }); + + const categories = {}; + + recentPosts.forEach(post => { + if (categories[post.category]) { + categories[post.category]++; + } else { + categories[post.category] = 1; + } + }); +} + +export async function predictAll() { + const allUsers = await User.find({}, { + fields: { + _id: true + } + }); + + allUsers.forEach(user => { + predictOne(user._id); + }); +} diff --git a/src/web/app/common/tags/post-menu.tag b/src/web/app/common/tags/post-menu.tag index 33895212b..be4468a21 100644 --- a/src/web/app/common/tags/post-menu.tag +++ b/src/web/app/common/tags/post-menu.tag @@ -2,6 +2,18 @@
+
+ + +