convert markdown content to html

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   A module to handle coding from internal to wire ActivityPub and back.
   """
   alias Pleroma.Activity
+  alias Pleroma.EarmarkRenderer
   alias Pleroma.FollowingRelationship
   alias Pleroma.Object
   alias Pleroma.Object.Containment
@@ -35,7 +36,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> fix_actor
     |> fix_url
     |> fix_attachments
-    |> fix_media_type
     |> fix_context
     |> fix_in_reply_to(options)
     |> fix_emoji
@@ -44,6 +44,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> fix_addressing
     |> fix_summary
     |> fix_type(options)
+    |> fix_content
   end
 
   def fix_summary(%{"summary" => nil} = object) do
@@ -358,11 +359,17 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def fix_type(object, _), do: object
 
-  defp fix_media_type(%{"mediaType" => _} = object) do
+  defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = object)
-    Map.put(object, "mediaType", "text/html")
+       when is_binary(content) do
+    html_content =
+      content
+      |> Earmark.as_html!(%Earmark.Options{renderer: EarmarkRenderer})
+      |> Pleroma.HTML.filter_tags()
+
+    Map.merge(object, %{"content" => html_content, "mediaType" => "text/html"})
   end
 
-  defp fix_media_type(object), do: object
+  defp fix_content(object), do: object
 
   defp mastodon_follow_hack(%{"id" => id, "actor" => follower_id}, followed) do
     with true <- id =~ "follows",

test/fixtures/tesla_mock/craigmaloney.json

@@ -0,0 +1,112 @@
+{
+    "@context": [
+        "https://www.w3.org/ns/activitystreams",
+        "https://w3id.org/security/v1",
+        {
+            "CacheFile": "pt:CacheFile",
+            "Hashtag": "as:Hashtag",
+            "Infohash": "pt:Infohash",
+            "RsaSignature2017": "https://w3id.org/security#RsaSignature2017",
+            "category": "sc:category",
+            "commentsEnabled": {
+                "@id": "pt:commentsEnabled",
+                "@type": "sc:Boolean"
+            },
+            "downloadEnabled": {
+                "@id": "pt:downloadEnabled",
+                "@type": "sc:Boolean"
+            },
+            "expires": "sc:expires",
+            "fps": {
+                "@id": "pt:fps",
+                "@type": "sc:Number"
+            },
+            "language": "sc:inLanguage",
+            "licence": "sc:license",
+            "originallyPublishedAt": "sc:datePublished",
+            "position": {
+                "@id": "pt:position",
+                "@type": "sc:Number"
+            },
+            "pt": "https://joinpeertube.org/ns#",
+            "sc": "http://schema.org#",
+            "sensitive": "as:sensitive",
+            "size": {
+                "@id": "pt:size",
+                "@type": "sc:Number"
+            },
+            "startTimestamp": {
+                "@id": "pt:startTimestamp",
+                "@type": "sc:Number"
+            },
+            "state": {
+                "@id": "pt:state",
+                "@type": "sc:Number"
+            },
+            "stopTimestamp": {
+                "@id": "pt:stopTimestamp",
+                "@type": "sc:Number"
+            },
+            "subtitleLanguage": "sc:subtitleLanguage",
+            "support": {
+                "@id": "pt:support",
+                "@type": "sc:Text"
+            },
+            "uuid": "sc:identifier",
+            "views": {
+                "@id": "pt:views",
+                "@type": "sc:Number"
+            },
+            "waitTranscoding": {
+                "@id": "pt:waitTranscoding",
+                "@type": "sc:Boolean"
+            }
+        },
+        {
+            "comments": {
+                "@id": "as:comments",
+                "@type": "@id"
+            },
+            "dislikes": {
+                "@id": "as:dislikes",
+                "@type": "@id"
+            },
+            "likes": {
+                "@id": "as:likes",
+                "@type": "@id"
+            },
+            "playlists": {
+                "@id": "pt:playlists",
+                "@type": "@id"
+            },
+            "shares": {
+                "@id": "as:shares",
+                "@type": "@id"
+            }
+        }
+    ],
+    "endpoints": {
+        "sharedInbox": "https://peertube.social/inbox"
+    },
+    "followers": "https://peertube.social/accounts/craigmaloney/followers",
+    "following": "https://peertube.social/accounts/craigmaloney/following",
+    "icon": {
+        "mediaType": "image/png",
+        "type": "Image",
+        "url": "https://peertube.social/lazy-static/avatars/87bd694b-95bc-4066-83f4-bddfcd2b9caa.png"
+    },
+    "id": "https://peertube.social/accounts/craigmaloney",
+    "inbox": "https://peertube.social/accounts/craigmaloney/inbox",
+    "name": "Craig Maloney",
+    "outbox": "https://peertube.social/accounts/craigmaloney/outbox",
+    "playlists": "https://peertube.social/accounts/craigmaloney/playlists",
+    "preferredUsername": "craigmaloney",
+    "publicKey": {
+        "id": "https://peertube.social/accounts/craigmaloney#main-key",
+        "owner": "https://peertube.social/accounts/craigmaloney",
+        "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA9qvGIYUW01yc8CCsrwxK\n5OXlV5s7EbNWY8tJr/p1oGuELZwAnG2XKxtdbvgcCT+YxL5uRXIdCFIIIKrzRFr/\nHfS0mOgNT9u3gu+SstCNgtatciT0RVP77yiC3b2NHq1NRRvvVhzQb4cpIWObIxqh\nb2ypDClTc7XaKtgmQCbwZlGyZMT+EKz/vustD6BlpGsglRkm7iES6s1PPGb1BU+n\nS94KhbS2DOFiLcXCVWt0QarokIIuKznp4+xP1axKyP+SkT5AHx08Nd5TYFb2C1Jl\nz0WD/1q0mAN62m7QrA3SQPUgB+wWD+S3Nzf7FwNPiP4srbBgxVEUnji/r9mQ6BXC\nrQIDAQAB\n-----END PUBLIC KEY-----"
+    },
+    "summary": null,
+    "type": "Person",
+    "url": "https://peertube.social/accounts/craigmaloney"
+}

test/fixtures/tesla_mock/hanimated.json

@@ -1 +0,0 @@
-{"type":"Person","id":"https://bittube.video/accounts/hanimated.moh","following":"https://bittube.video/accounts/hanimated.moh/following","followers":"https://bittube.video/accounts/hanimated.moh/followers","playlists":"https://bittube.video/accounts/hanimated.moh/playlists","inbox":"https://bittube.video/accounts/hanimated.moh/inbox","outbox":"https://bittube.video/accounts/hanimated.moh/outbox","preferredUsername":"hanimated.moh","url":"https://bittube.video/accounts/hanimated.moh","name":"Nosat","endpoints":{"sharedInbox":"https://bittube.video/inbox"},"publicKey":{"id":"https://bittube.video/accounts/hanimated.moh#main-key","owner":"https://bittube.video/accounts/hanimated.moh","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwuoQT+4uyAboQcf/okCM\nFqUS/LuqFc2888OSKZFAz00Op/dyOB/pkr1+QLxbl8ZGiUWhmnmhNwmmd3tbhSsC\nvLv9Mz/YaWQPYLfRS/s/7iIxdniC4lo/YgicOrzcvetHmk1feOg5vb5/yc+bgUSm\nOk+L4azqXP9GmZyofzvufT65bUmzQRFXP19eL55YZWvZDaC81QAfRXsqtCqbehtF\nQNOjGhnl6a7Kfe8KprRDPV/3WvvFjftnNO2qenIIOFLLeznkQ0ELP6lyb9pvv/1C\n2/GRh2BwmgVlCTw1kTxLSdj80BFX5P8AudSiIx079lVkhamEhzsNLkMpQFqWAAlg\nrQIDAQAB\n-----END PUBLIC KEY-----"},"icon":{"type":"Image","mediaType":"image/jpeg","url":"https://bittube.video/lazy-static/avatars/84b8acc3-e48b-4642-a9f4-360a4499579b.jpg"},"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1",{"RsaSignature2017":"https://w3id.org/security#RsaSignature2017","pt":"https://joinpeertube.org/ns#","sc":"http://schema.org#","Hashtag":"as:Hashtag","uuid":"sc:identifier","category":"sc:category","licence":"sc:license","subtitleLanguage":"sc:subtitleLanguage","sensitive":"as:sensitive","language":"sc:inLanguage","expires":"sc:expires","CacheFile":"pt:CacheFile","Infohash":"pt:Infohash","originallyPublishedAt":"sc:datePublished","views":{"@type":"sc:Number","@id":"pt:views"},"state":{"@type":"sc:Number","@id":"pt:state"},"size":{"@type":"sc:Number","@id":"pt:size"},"fps":{"@type":"sc:Number","@id":"pt:fps"},"startTimestamp":{"@type":"sc:Number","@id":"pt:startTimestamp"},"stopTimestamp":{"@type":"sc:Number","@id":"pt:stopTimestamp"},"position":{"@type":"sc:Number","@id":"pt:position"},"commentsEnabled":{"@type":"sc:Boolean","@id":"pt:commentsEnabled"},"downloadEnabled":{"@type":"sc:Boolean","@id":"pt:downloadEnabled"},"waitTranscoding":{"@type":"sc:Boolean","@id":"pt:waitTranscoding"},"support":{"@type":"sc:Text","@id":"pt:support"}},{"likes":{"@id":"as:likes","@type":"@id"},"dislikes":{"@id":"as:dislikes","@type":"@id"},"playlists":{"@id":"pt:playlists","@type":"@id"},"shares":{"@id":"as:shares","@type":"@id"},"comments":{"@id":"as:comments","@type":"@id"}}],"summary":null}

test/fixtures/tesla_mock/peertube-social.json

@@ -0,0 +1,234 @@
+{
+    "@context": [
+        "https://www.w3.org/ns/activitystreams",
+        "https://w3id.org/security/v1",
+        {
+            "CacheFile": "pt:CacheFile",
+            "Hashtag": "as:Hashtag",
+            "Infohash": "pt:Infohash",
+            "RsaSignature2017": "https://w3id.org/security#RsaSignature2017",
+            "category": "sc:category",
+            "commentsEnabled": {
+                "@id": "pt:commentsEnabled",
+                "@type": "sc:Boolean"
+            },
+            "downloadEnabled": {
+                "@id": "pt:downloadEnabled",
+                "@type": "sc:Boolean"
+            },
+            "expires": "sc:expires",
+            "fps": {
+                "@id": "pt:fps",
+                "@type": "sc:Number"
+            },
+            "language": "sc:inLanguage",
+            "licence": "sc:license",
+            "originallyPublishedAt": "sc:datePublished",
+            "position": {
+                "@id": "pt:position",
+                "@type": "sc:Number"
+            },
+            "pt": "https://joinpeertube.org/ns#",
+            "sc": "http://schema.org#",
+            "sensitive": "as:sensitive",
+            "size": {
+                "@id": "pt:size",
+                "@type": "sc:Number"
+            },
+            "startTimestamp": {
+                "@id": "pt:startTimestamp",
+                "@type": "sc:Number"
+            },
+            "state": {
+                "@id": "pt:state",
+                "@type": "sc:Number"
+            },
+            "stopTimestamp": {
+                "@id": "pt:stopTimestamp",
+                "@type": "sc:Number"
+            },
+            "subtitleLanguage": "sc:subtitleLanguage",
+            "support": {
+                "@id": "pt:support",
+                "@type": "sc:Text"
+            },
+            "uuid": "sc:identifier",
+            "views": {
+                "@id": "pt:views",
+                "@type": "sc:Number"
+            },
+            "waitTranscoding": {
+                "@id": "pt:waitTranscoding",
+                "@type": "sc:Boolean"
+            }
+        },
+        {
+            "comments": {
+                "@id": "as:comments",
+                "@type": "@id"
+            },
+            "dislikes": {
+                "@id": "as:dislikes",
+                "@type": "@id"
+            },
+            "likes": {
+                "@id": "as:likes",
+                "@type": "@id"
+            },
+            "playlists": {
+                "@id": "pt:playlists",
+                "@type": "@id"
+            },
+            "shares": {
+                "@id": "as:shares",
+                "@type": "@id"
+            }
+        }
+    ],
+    "attributedTo": [
+        {
+            "id": "https://peertube.social/accounts/craigmaloney",
+            "type": "Person"
+        },
+        {
+            "id": "https://peertube.social/video-channels/9909c7d9-6b5b-4aae-9164-c1af7229c91c",
+            "type": "Group"
+        }
+    ],
+    "category": {
+        "identifier": "15",
+        "name": "Science & Technology"
+    },
+    "cc": [
+        "https://peertube.social/accounts/craigmaloney/followers"
+    ],
+    "comments": "https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe/comments",
+    "commentsEnabled": true,
+    "content": "Support this and our other Michigan!/usr/group videos and meetings. Learn more at http://mug.org/membership\n\nTwenty Years in Jail: FreeBSD's Jails, Then and Now\n\nJails started as a limited virtualization system, but over the last two years they've...",
+    "dislikes": "https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe/dislikes",
+    "downloadEnabled": true,
+    "duration": "PT5151S",
+    "icon": {
+        "height": 122,
+        "mediaType": "image/jpeg",
+        "type": "Image",
+        "url": "https://peertube.social/static/thumbnails/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe.jpg",
+        "width": 223
+    },
+    "id": "https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe",
+    "language": {
+        "identifier": "en",
+        "name": "English"
+    },
+    "licence": {
+        "identifier": "1",
+        "name": "Attribution"
+    },
+    "likes": "https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe/likes",
+    "mediaType": "text/markdown",
+    "name": "Twenty Years in Jail: FreeBSD's Jails, Then and Now",
+    "originallyPublishedAt": "2019-08-13T00:00:00.000Z",
+    "published": "2020-02-12T01:06:08.054Z",
+    "sensitive": false,
+    "shares": "https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe/announces",
+    "state": 1,
+    "subtitleLanguage": [],
+    "support": "Learn more at http://mug.org",
+    "tag": [
+        {
+            "name": "linux",
+            "type": "Hashtag"
+        },
+        {
+            "name": "mug.org",
+            "type": "Hashtag"
+        },
+        {
+            "name": "open",
+            "type": "Hashtag"
+        },
+        {
+            "name": "oss",
+            "type": "Hashtag"
+        },
+        {
+            "name": "source",
+            "type": "Hashtag"
+        }
+    ],
+    "to": [
+        "https://www.w3.org/ns/activitystreams#Public"
+    ],
+    "type": "Video",
+    "updated": "2020-02-15T15:01:09.474Z",
+    "url": [
+        {
+            "href": "https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe",
+            "mediaType": "text/html",
+            "type": "Link"
+        },
+        {
+            "fps": 30,
+            "height": 240,
+            "href": "https://peertube.social/static/webseed/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-240.mp4",
+            "mediaType": "video/mp4",
+            "size": 119465800,
+            "type": "Link"
+        },
+        {
+            "height": 240,
+            "href": "https://peertube.social/static/torrents/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-240.torrent",
+            "mediaType": "application/x-bittorrent",
+            "type": "Link"
+        },
+        {
+            "height": 240,
+            "href": "magnet:?xs=https%3A%2F%2Fpeertube.social%2Fstatic%2Ftorrents%2F278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-240.torrent&xt=urn:btih:b3365331a8543bf48d09add56d7fe4b1cbbb5659&dn=Twenty+Years+in+Jail%3A+FreeBSD's+Jails%2C+Then+and+Now&tr=wss%3A%2F%2Fpeertube.social%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fpeertube.social%2Ftracker%2Fannounce&ws=https%3A%2F%2Fpeertube.social%2Fstatic%2Fwebseed%2F278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-240.mp4",
+            "mediaType": "application/x-bittorrent;x-scheme-handler/magnet",
+            "type": "Link"
+        },
+        {
+            "fps": 30,
+            "height": 360,
+            "href": "https://peertube.social/static/webseed/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-360.mp4",
+            "mediaType": "video/mp4",
+            "size": 143930318,
+            "type": "Link"
+        },
+        {
+            "height": 360,
+            "href": "https://peertube.social/static/torrents/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-360.torrent",
+            "mediaType": "application/x-bittorrent",
+            "type": "Link"
+        },
+        {
+            "height": 360,
+            "href": "magnet:?xs=https%3A%2F%2Fpeertube.social%2Fstatic%2Ftorrents%2F278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-360.torrent&xt=urn:btih:0d37b23c98cb0d89e28b5dc8f49b3c97a041e569&dn=Twenty+Years+in+Jail%3A+FreeBSD's+Jails%2C+Then+and+Now&tr=wss%3A%2F%2Fpeertube.social%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fpeertube.social%2Ftracker%2Fannounce&ws=https%3A%2F%2Fpeertube.social%2Fstatic%2Fwebseed%2F278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-360.mp4",
+            "mediaType": "application/x-bittorrent;x-scheme-handler/magnet",
+            "type": "Link"
+        },
+        {
+            "fps": 30,
+            "height": 480,
+            "href": "https://peertube.social/static/webseed/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-480.mp4",
+            "mediaType": "video/mp4",
+            "size": 130530754,
+            "type": "Link"
+        },
+        {
+            "height": 480,
+            "href": "https://peertube.social/static/torrents/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-480.torrent",
+            "mediaType": "application/x-bittorrent",
+            "type": "Link"
+        },
+        {
+            "height": 480,
+            "href": "magnet:?xs=https%3A%2F%2Fpeertube.social%2Fstatic%2Ftorrents%2F278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-480.torrent&xt=urn:btih:3a13ff822ad9494165eff6167183ddaaabc1372a&dn=Twenty+Years+in+Jail%3A+FreeBSD's+Jails%2C+Then+and+Now&tr=wss%3A%2F%2Fpeertube.social%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fpeertube.social%2Ftracker%2Fannounce&ws=https%3A%2F%2Fpeertube.social%2Fstatic%2Fwebseed%2F278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-480.mp4",
+            "mediaType": "application/x-bittorrent;x-scheme-handler/magnet",
+            "type": "Link"
+        }
+    ],
+    "uuid": "278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe",
+    "views": 2,
+    "waitTranscoding": false
+}

test/support/http_request_mock.ex

@@ -308,19 +308,19 @@ defmodule HttpRequestMock do
      }}
   end
 
-  def get("https://bittube.video/accounts/hanimated.moh", _, _, _) do
+  def get("https://peertube.social/accounts/craigmaloney", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/hanimated.json")
+       body: File.read!("test/fixtures/tesla_mock/craigmaloney.json")
      }}
   end
 
-  def get("https://bittube.video/videos/watch/2aad7dfb-5c75-4ee6-a9ed-08436af0558b", _, _, _) do
+  def get("https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/bittube-video.json")
+       body: File.read!("test/fixtures/tesla_mock/peertube-social.json")
      }}
   end
 

test/web/activity_pub/transmogrifier_test.exs

@@ -1221,11 +1221,20 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
       :error = Transmogrifier.handle_incoming(data)
     end
 
-    test "it remaps mediaType of object" do
+    test "it converts content of object to html" do
-      {:ok, object} =
+      object_id = "https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe"
-        Fetcher.fetch_object_from_id(
+
-          "https://bittube.video/videos/watch/2aad7dfb-5c75-4ee6-a9ed-08436af0558b"
+      {:ok, %{"content" => content_markdown}} =
-        )
+        Fetcher.fetch_and_contain_remote_object_from_id(object_id)
+
+      {:ok, %Pleroma.Object{data: %{"content" => content}} = object} =
+        Fetcher.fetch_object_from_id(object_id)
+
+      assert content_markdown ==
+               "Support this and our other Michigan!/usr/group videos and meetings. Learn more at http://mug.org/membership\n\nTwenty Years in Jail: FreeBSD's Jails, Then and Now\n\nJails started as a limited virtualization system, but over the last two years they've..."
+
+      assert content ==
+               "<p>Support this and our other Michigan!/usr/group videos and meetings. Learn more at <a href=\"http://mug.org/membership\">http://mug.org/membership</a></p><p>Twenty Years in Jail: FreeBSD’s Jails, Then and Now</p><p>Jails started as a limited virtualization system, but over the last two years they’ve…</p>"
 
       assert object.data["mediaType"] == "text/html"
     end