Add angle bracket covered url syntax to mfm (#4483)

* Add angle bracket covered url syntax to mfm

* Fix path

* Fix match

* Fix index

src/mfm/fromHtml.ts

@@ -1,5 +1,6 @@
 import { parseFragment, DefaultTreeDocumentFragment } from 'parse5';
 import { URL } from 'url';
+import { urlRegex } from './prelude';
 
 export function fromHtml(html: string): string {
 	if (html == null) return null;
@@ -14,7 +15,7 @@ export function fromHtml(html: string): string {
 
 	return text.trim();
 
-	function getText(node: any) {
+	function getText(node: any): string {
 		if (node.nodeName == '#text') return node.value;
 
 		if (node.childNodes) {
@@ -41,7 +42,7 @@ export function fromHtml(html: string): string {
 
 				// ハッシュタグ / hrefがない / txtがURL
 				if ((rel && rel.value.match('tag') !== null) || !href || href.value == txt) {
-					text += txt;
+					text += txt.match(urlRegex) ? txt : `<${txt}>`;
 				// メンション
 				} else if (txt.startsWith('@') && !(rel && rel.value.match(/^me /))) {
 					const part = txt.split('@');

src/mfm/language.ts

@@ -1,5 +1,5 @@
 import * as P from 'parsimmon';
-import { createLeaf, createTree } from './types';
+import { createLeaf, createTree, urlRegex } from './prelude';
 import { takeWhile, cumulativeSum } from '../prelude/array';
 import parseAcct from '../misc/acct/parse';
 import { toUnicode } from 'punycode';
@@ -154,9 +154,16 @@ export const mfmLanguage = P.createLanguage({
 	url: () => {
 		return P((input, i) => {
 			const text = input.substr(i);
-			const match = text.match(/^https?:\/\/[\w\/:%#@\$&\?!\(\)\[\]~\.,=\+\-]+/);
+			const match = text.match(urlRegex);
-			if (!match) return P.makeFailure(i, 'not a url');
+			let url: string;
-			let url = match[0];
+			if (!match) {
+				const match = text.match(/^<(https?:\/\/.*?)>/);
+				if (!match)
+					return P.makeFailure(i, 'not a url');
+				url = match[1];
+				i += 2;
+			} else
+				url = match[0];
 			url = removeOrphanedBrackets(url);
 			if (url.endsWith('.')) url = url.substr(0, url.lastIndexOf('.'));
 			if (url.endsWith(',')) url = url.substr(0, url.lastIndexOf(','));

src/mfm/normalize.ts

@@ -1,6 +1,6 @@
 import * as A from '../prelude/array';
 import * as S from '../prelude/string';
-import { MfmForest, MfmTree } from './types';
+import { MfmForest, MfmTree } from './prelude';
 import { createTree, createLeaf } from '../prelude/tree';
 
 function isEmptyTextTree(t: MfmTree): boolean {

src/mfm/parse.ts

@@ -1,5 +1,5 @@
 import { mfmLanguage } from './language';
-import { MfmForest } from './types';
+import { MfmForest } from './prelude';
 import { normalize } from './normalize';
 
 export function parse(source: string): MfmForest {

src/mfm/prelude.ts

@@ -35,3 +35,5 @@ export function createLeaf(type: string, props: any): MfmTree {
 export function createTree(type: string, children: MfmForest, props: any): MfmTree {
 	return T.createTree({ type, props }, children);
 }
+
+export const urlRegex = /^https?:\/\/[\w\/:%#@\$&\?!\(\)\[\]~\.,=\+\-]+/;

src/mfm/toHtml.ts

@@ -2,7 +2,7 @@ import { JSDOM } from 'jsdom';
 import config from '../config';
 import { INote } from '../models/note';
 import { intersperse } from '../prelude/array';
-import { MfmForest, MfmTree } from './types';
+import { MfmForest, MfmTree } from './prelude';
 
 export function toHtml(tokens: MfmForest, mentionedRemoteUsers: INote['mentionedRemoteUsers'] = []) {
 	if (tokens == null) {

src/misc/extract-emojis.ts

@@ -1,4 +1,4 @@
-import { EmojiNode, MfmForest } from '../mfm/types';
+import { EmojiNode, MfmForest } from '../mfm/prelude';
 import { preorderF } from '../prelude/tree';
 import { unique } from '../prelude/array';
 

src/misc/extract-hashtags.ts

@@ -1,4 +1,4 @@
-import { HashtagNode, MfmForest } from '../mfm/types';
+import { HashtagNode, MfmForest } from '../mfm/prelude';
 import { preorderF } from '../prelude/tree';
 import { unique } from '../prelude/array';
 

src/misc/extract-mentions.ts

@@ -1,6 +1,6 @@
 // test is located in test/extract-mentions
 
-import { MentionNode, MfmForest } from '../mfm/types';
+import { MentionNode, MfmForest } from '../mfm/prelude';
 import { preorderF } from '../prelude/tree';
 
 export default function(mfmForest: MfmForest): MentionNode['props'][] {

test/mfm.ts

@@ -12,7 +12,7 @@ import * as assert from 'assert';
 
 import { parse, parsePlain } from '../src/mfm/parse';
 import { toHtml } from '../src/mfm/toHtml';
-import { createTree as tree, createLeaf as leaf, MfmTree } from '../src/mfm/types';
+import { createTree as tree, createLeaf as leaf, MfmTree } from '../src/mfm/prelude';
 import { removeOrphanedBrackets } from '../src/mfm/language';
 
 function text(text: string): MfmTree {
@@ -840,6 +840,20 @@ describe('MFM', () => {
 					text(')')
 				]);
 			});
+
+			it('ignore non-ascii characters contained url without angle brackets', () => {
+				const tokens = parse('https://大石泉すき.example.com');
+				assert.deepStrictEqual(tokens, [
+					text('https://大石泉すき.example.com')
+				]);
+			});
+
+			it('match non-ascii characters contained url with angle brackets', () => {
+				const tokens = parse('<https://大石泉すき.example.com>');
+				assert.deepStrictEqual(tokens, [
+					leaf('url', { url: 'https://大石泉すき.example.com' })
+				]);
+			});
 		});
 
 		describe('link', () => {