From 38b2db297b3207607072347b408dc7eacbac600e Mon Sep 17 00:00:00 2001
From: stwf <steven.fuchs@dockyard.com>
Date: Mon, 14 Sep 2020 13:18:11 -0400
Subject: [PATCH 01/12] search indexing metadata respects discoverable flag

---
 lib/pleroma/web/metadata/restrict_indexing.ex |  7 +++----
 test/web/metadata/metadata_test.exs           | 19 +++++++++++++++++--
 test/web/metadata/restrict_indexing_test.exs  |  8 +++++++-
 3 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/lib/pleroma/web/metadata/restrict_indexing.ex b/lib/pleroma/web/metadata/restrict_indexing.ex
index f15607896..a1dcb6e15 100644
--- a/lib/pleroma/web/metadata/restrict_indexing.ex
+++ b/lib/pleroma/web/metadata/restrict_indexing.ex
@@ -10,7 +10,9 @@ defmodule Pleroma.Web.Metadata.Providers.RestrictIndexing do
   """
 
   @impl true
-  def build_tags(%{user: %{local: false}}) do
+  def build_tags(%{user: %{local: true, discoverable: true}}), do: []
+
+  def build_tags(_) do
     [
       {:meta,
        [
@@ -19,7 +21,4 @@ defmodule Pleroma.Web.Metadata.Providers.RestrictIndexing do
        ], []}
     ]
   end
-
-  @impl true
-  def build_tags(%{user: %{local: true}}), do: []
 end
diff --git a/test/web/metadata/metadata_test.exs b/test/web/metadata/metadata_test.exs
index 9d3121b7b..fe3009628 100644
--- a/test/web/metadata/metadata_test.exs
+++ b/test/web/metadata/metadata_test.exs
@@ -18,17 +18,32 @@ defmodule Pleroma.Web.MetadataTest do
     test "for local user" do
       user = insert(:user)
 
+      assert Pleroma.Web.Metadata.build_tags(%{user: user}) =~
+               "<meta content=\"noindex, noarchive\" name=\"robots\">"
+    end
+
+    test "for local user set to discoverable" do
+      user = insert(:user, discoverable: true)
+
       refute Pleroma.Web.Metadata.build_tags(%{user: user}) =~
                "<meta content=\"noindex, noarchive\" name=\"robots\">"
     end
   end
 
   describe "no metadata for private instances" do
-    test "for local user" do
+    test "for local user set to discoverable" do
       clear_config([:instance, :public], false)
-      user = insert(:user, bio: "This is my secret fedi account bio")
+      user = insert(:user, bio: "This is my secret fedi account bio", discoverable: true)
 
       assert "" = Pleroma.Web.Metadata.build_tags(%{user: user})
     end
+
+    test "search exclusion metadata is included" do
+      clear_config([:instance, :public], false)
+      user = insert(:user, bio: "This is my secret fedi account bio")
+
+      assert "<meta content=\"noindex, noarchive\" name=\"robots\">" ==
+               Pleroma.Web.Metadata.build_tags(%{user: user})
+    end
   end
 end
diff --git a/test/web/metadata/restrict_indexing_test.exs b/test/web/metadata/restrict_indexing_test.exs
index aad0bac42..6b3a65372 100644
--- a/test/web/metadata/restrict_indexing_test.exs
+++ b/test/web/metadata/restrict_indexing_test.exs
@@ -14,8 +14,14 @@ defmodule Pleroma.Web.Metadata.Providers.RestrictIndexingTest do
 
     test "for local user" do
       assert Pleroma.Web.Metadata.Providers.RestrictIndexing.build_tags(%{
-               user: %Pleroma.User{local: true}
+               user: %Pleroma.User{local: true, discoverable: true}
              }) == []
     end
+
+    test "for local user when discoverable is false" do
+      assert Pleroma.Web.Metadata.Providers.RestrictIndexing.build_tags(%{
+               user: %Pleroma.User{local: true, discoverable: false}
+             }) == [{:meta, [name: "robots", content: "noindex, noarchive"], []}]
+    end
   end
 end

From f900a40d5dc4285977bd92f3792ad04a2f34ddcf Mon Sep 17 00:00:00 2001
From: stwf <steven.fuchs@dockyard.com>
Date: Mon, 14 Sep 2020 13:55:49 -0400
Subject: [PATCH 02/12] fix credo warning

---
 test/web/metadata/metadata_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/web/metadata/metadata_test.exs b/test/web/metadata/metadata_test.exs
index fe3009628..054844597 100644
--- a/test/web/metadata/metadata_test.exs
+++ b/test/web/metadata/metadata_test.exs
@@ -42,7 +42,7 @@ defmodule Pleroma.Web.MetadataTest do
       clear_config([:instance, :public], false)
       user = insert(:user, bio: "This is my secret fedi account bio")
 
-      assert "<meta content=\"noindex, noarchive\" name=\"robots\">" ==
+      assert ~s(<meta content="noindex, noarchive" name="robots">) ==
                Pleroma.Web.Metadata.build_tags(%{user: user})
     end
   end

From 582ad5d4e1587b3dba9d879bd68dd9a315c8446e Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Sun, 30 Aug 2020 15:15:14 +0300
Subject: [PATCH 03/12] AdminAPI: Allow to modify Terms of Service and Instance
 Panel via Admin API

---
 CHANGELOG.md                                  |   6 +
 docs/API/admin_api.md                         |  42 +++++++
 .../instance_document_controller.ex           |  37 ++++++
 .../admin/instance_document_operation.ex      | 108 +++++++++++++++++
 lib/pleroma/web/instance_document.ex          |  62 ++++++++++
 lib/pleroma/web/router.ex                     |   4 +
 test/fixtures/custom_instance_panel.html      |   1 +
 .../instance_document_controller_test.exs     | 112 ++++++++++++++++++
 8 files changed, 372 insertions(+)
 create mode 100644 lib/pleroma/web/admin_api/controllers/instance_document_controller.ex
 create mode 100644 lib/pleroma/web/api_spec/operations/admin/instance_document_operation.ex
 create mode 100644 lib/pleroma/web/instance_document.ex
 create mode 100644 test/fixtures/custom_instance_panel.html
 create mode 100644 test/web/admin_api/controllers/instance_document_controller_test.exs

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f7a372e11..1ec6c11cc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -41,6 +41,12 @@ switched to a new configuration mechanism, however it was not officially removed
 ### Added
 - Rich media failure tracking (along with `:failure_backoff` option).
 
+<details>
+  <summary>Admin API Changes</summary>
+
+- Add `PATCH /api/pleroma/admin/instance_document/:document_name` to modify the Terms of Service and Instance Panel HTML pages via Admin API
+</details>
+
 ### Fixed
 - Default HTTP adapter not respecting pool setting, leading to possible OOM.
 - Fixed uploading webp images when the Exiftool Upload Filter is enabled by skipping them
diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md
index bc96abbf0..eba92dd1f 100644
--- a/docs/API/admin_api.md
+++ b/docs/API/admin_api.md
@@ -1455,3 +1455,45 @@ Loads json generated from `config/descriptions.exs`.
   "unread": false
 }
 ```
+
+## `GET /api/pleroma/admin/instance_document/:document_name`
+
+### Gets an instance document
+
+- Authentication: required
+
+- Response:
+
+``` json
+{
+  "url": "https://example.com/instance/panel.html"
+}
+```
+
+## `PATCH /api/pleroma/admin/instance_document/:document_name`
+- Params:
+  - `file` (the file to be uploaded, using multipart form data.)
+
+### Updates an instance document
+
+- Authentication: required
+
+- Response:
+
+``` json
+{
+  "url": "https://example.com/instance/panel.html"
+}
+```
+
+## `DELETE /api/pleroma/admin/instance_document/:document_name`
+
+### Delete an instance document
+
+- Response:
+
+``` json
+{
+  "url": "https://example.com/instance/panel.html"
+}
+```
diff --git a/lib/pleroma/web/admin_api/controllers/instance_document_controller.ex b/lib/pleroma/web/admin_api/controllers/instance_document_controller.ex
new file mode 100644
index 000000000..2144e44ac
--- /dev/null
+++ b/lib/pleroma/web/admin_api/controllers/instance_document_controller.ex
@@ -0,0 +1,37 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.InstanceDocumentController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.Web.InstanceDocument
+
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+
+  action_fallback(Pleroma.Web.AdminAPI.FallbackController)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.InstanceDocumentOperation
+
+  plug(OAuthScopesPlug, %{scopes: ["read"], admin: true} when action == :show)
+  plug(OAuthScopesPlug, %{scopes: ["write"], admin: true} when action in [:update, :delete])
+
+  def show(conn, %{name: document_name}) do
+    with {:ok, url} <- InstanceDocument.get(document_name) do
+      json(conn, %{"url" => url})
+    end
+  end
+
+  def update(%{body_params: %{file: file}} = conn, %{name: document_name}) do
+    with {:ok, url} <- InstanceDocument.put(document_name, file.path) do
+      json(conn, %{"url" => url})
+    end
+  end
+
+  def delete(conn, %{name: document_name}) do
+    with :ok <- InstanceDocument.delete(document_name) do
+      json(conn, %{})
+    end
+  end
+end
diff --git a/lib/pleroma/web/api_spec/operations/admin/instance_document_operation.ex b/lib/pleroma/web/api_spec/operations/admin/instance_document_operation.ex
new file mode 100644
index 000000000..e0eb993fb
--- /dev/null
+++ b/lib/pleroma/web/api_spec/operations/admin/instance_document_operation.ex
@@ -0,0 +1,108 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.InstanceDocumentOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Helpers
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def show_operation do
+    %Operation{
+      tags: ["Admin", "InstanceDocument"],
+      summary: "Get the instance document",
+      operationId: "AdminAPI.InstanceDocumentController.show",
+      security: [%{"oAuth" => ["read"]}],
+      parameters: [
+        Operation.parameter(:name, :path, %Schema{type: :string}, "The document name",
+          required: true
+        )
+        | Helpers.admin_api_params()
+      ],
+      responses: %{
+        200 => Operation.response("InstanceDocument", "application/json", instance_document()),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        403 => Operation.response("Forbidden", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  def update_operation do
+    %Operation{
+      tags: ["Admin", "InstanceDocument"],
+      summary: "Update the instance document",
+      operationId: "AdminAPI.InstanceDocumentController.update",
+      security: [%{"oAuth" => ["write"]}],
+      requestBody: Helpers.request_body("Parameters", update_request()),
+      parameters: [
+        Operation.parameter(:name, :path, %Schema{type: :string}, "The document name",
+          required: true
+        )
+        | Helpers.admin_api_params()
+      ],
+      responses: %{
+        200 => Operation.response("InstanceDocument", "application/json", instance_document()),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        403 => Operation.response("Forbidden", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp update_request do
+    %Schema{
+      title: "UpdateRequest",
+      description: "POST body for uploading the file",
+      type: :object,
+      required: [:file],
+      properties: %{
+        file: %Schema{
+          type: :string,
+          format: :binary,
+          description: "The file to be uploaded, using multipart form data."
+        }
+      }
+    }
+  end
+
+  def delete_operation do
+    %Operation{
+      tags: ["Admin", "InstanceDocument"],
+      summary: "Get the instance document",
+      operationId: "AdminAPI.InstanceDocumentController.delete",
+      security: [%{"oAuth" => ["write"]}],
+      parameters: [
+        Operation.parameter(:name, :path, %Schema{type: :string}, "The document name",
+          required: true
+        )
+        | Helpers.admin_api_params()
+      ],
+      responses: %{
+        200 => Operation.response("InstanceDocument", "application/json", instance_document()),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        403 => Operation.response("Forbidden", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp instance_document do
+    %Schema{
+      title: "InstanceDocument",
+      type: :object,
+      properties: %{
+        url: %Schema{type: :string}
+      },
+      example: %{
+        "url" => "https://example.com/static/terms-of-service.html"
+      }
+    }
+  end
+end
diff --git a/lib/pleroma/web/instance_document.ex b/lib/pleroma/web/instance_document.ex
new file mode 100644
index 000000000..969a44e41
--- /dev/null
+++ b/lib/pleroma/web/instance_document.ex
@@ -0,0 +1,62 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.InstanceDocument do
+  alias Pleroma.Config
+  alias Pleroma.Web.Endpoint
+
+  @instance_documents %{
+    "terms-of-service" => "/static/terms-of-service.html",
+    "instance-panel" => "/instance/panel.html"
+  }
+
+  @spec get(String.t()) :: {:ok, String.t()} | {:error, atom()}
+  def get(document_name) do
+    case Map.fetch(@instance_documents, document_name) do
+      {:ok, path} -> {:ok, Path.join(Endpoint.url(), path)}
+      _ -> {:error, :not_found}
+    end
+  end
+
+  @spec put(String.t(), String.t()) :: {:ok, String.t()} | {:error, atom()}
+  def put(document_name, origin_path) do
+    with {_, {:ok, destination_path}} <-
+           {:instance_document, Map.fetch(@instance_documents, document_name)},
+         :ok <- put_file(origin_path, destination_path) do
+      {:ok, Path.join(Endpoint.url(), destination_path)}
+    else
+      {:instance_document, :error} -> {:error, :not_found}
+      error -> error
+    end
+  end
+
+  @spec delete(String.t()) :: :ok | {:error, atom()}
+  def delete(document_name) do
+    with {_, {:ok, path}} <- {:instance_document, Map.fetch(@instance_documents, document_name)},
+         instance_static_dir_path <- instance_static_dir(path),
+         :ok <- File.rm(instance_static_dir_path) do
+      :ok
+    else
+      {:instance_document, :error} -> {:error, :not_found}
+      {:error, :enoent} -> {:error, :not_found}
+      error -> error
+    end
+  end
+
+  defp put_file(origin_path, destination_path) do
+    with destination <- instance_static_dir(destination_path),
+         {_, :ok} <- {:mkdir_p, File.mkdir_p(Path.dirname(destination))},
+         {_, {:ok, _}} <- {:copy, File.copy(origin_path, destination)} do
+      :ok
+    else
+      {error, _} -> {:error, error}
+    end
+  end
+
+  defp instance_static_dir(filename) do
+    [:instance, :static_dir]
+    |> Config.get!()
+    |> Path.join(filename)
+  end
+end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index e4440d442..a4a58c2c4 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -182,6 +182,10 @@ defmodule Pleroma.Web.Router do
 
     get("/instances/:instance/statuses", AdminAPIController, :list_instance_statuses)
 
+    get("/instance_document/:name", InstanceDocumentController, :show)
+    patch("/instance_document/:name", InstanceDocumentController, :update)
+    delete("/instance_document/:name", InstanceDocumentController, :delete)
+
     patch("/users/confirm_email", AdminAPIController, :confirm_email)
     patch("/users/resend_confirmation_email", AdminAPIController, :resend_confirmation_email)
 
diff --git a/test/fixtures/custom_instance_panel.html b/test/fixtures/custom_instance_panel.html
new file mode 100644
index 000000000..6371a1e43
--- /dev/null
+++ b/test/fixtures/custom_instance_panel.html
@@ -0,0 +1 @@
+<h2>Custom instance panel</h2>
\ No newline at end of file
diff --git a/test/web/admin_api/controllers/instance_document_controller_test.exs b/test/web/admin_api/controllers/instance_document_controller_test.exs
new file mode 100644
index 000000000..60dcc9dff
--- /dev/null
+++ b/test/web/admin_api/controllers/instance_document_controller_test.exs
@@ -0,0 +1,112 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.InstanceDocumentControllerTest do
+  use Pleroma.Web.ConnCase, async: true
+  import Pleroma.Factory
+  alias Pleroma.Config
+
+  @dir "test/tmp/instance_static"
+  @default_instance_panel ~s(<p>Welcome to <a href="https://pleroma.social" target="_blank">Pleroma!</a></p>)
+
+  setup do
+    File.mkdir_p!(@dir)
+    on_exit(fn -> File.rm_rf(@dir) end)
+  end
+
+  setup do: clear_config([:instance, :static_dir], @dir)
+
+  setup do
+    admin = insert(:user, is_admin: true)
+    token = insert(:oauth_admin_token, user: admin)
+
+    conn =
+      build_conn()
+      |> assign(:user, admin)
+      |> assign(:token, token)
+
+    {:ok, %{admin: admin, token: token, conn: conn}}
+  end
+
+  describe "GET /api/pleroma/admin/instance_document/:name" do
+    test "return the instance document url", %{conn: conn} do
+      conn = get(conn, "/api/pleroma/admin/instance_document/instance-panel")
+
+      assert %{"url" => url} = json_response_and_validate_schema(conn, 200)
+      index = get(build_conn(), url)
+      response = html_response(index, 200)
+      assert String.contains?(response, @default_instance_panel)
+    end
+
+    test "it returns 403 if requested by a non-admin" do
+      non_admin_user = insert(:user)
+      token = insert(:oauth_token, user: non_admin_user)
+
+      conn =
+        build_conn()
+        |> assign(:user, non_admin_user)
+        |> assign(:token, token)
+        |> get("/api/pleroma/admin/instance_document/instance-panel")
+
+      assert json_response(conn, :forbidden)
+    end
+
+    test "it returns 404 if the instance document with the given name doesn't exist", %{
+      conn: conn
+    } do
+      conn = get(conn, "/api/pleroma/admin/instance_document/1234")
+
+      assert json_response_and_validate_schema(conn, 404)
+    end
+  end
+
+  describe "PATCH /api/pleroma/admin/instance_document/:name" do
+    test "uploads the instance document", %{conn: conn} do
+      image = %Plug.Upload{
+        content_type: "text/html",
+        path: Path.absname("test/fixtures/custom_instance_panel.html"),
+        filename: "custom_instance_panel.html"
+      }
+
+      conn =
+        conn
+        |> put_req_header("content-type", "multipart/form-data")
+        |> patch("/api/pleroma/admin/instance_document/instance-panel", %{
+          "file" => image
+        })
+
+      assert %{"url" => url} = json_response_and_validate_schema(conn, 200)
+      index = get(build_conn(), url)
+      assert html_response(index, 200) == "<h2>Custom instance panel</h2>"
+    end
+  end
+
+  describe "DELETE /api/pleroma/admin/instance_document/:name" do
+    test "deletes the instance document", %{conn: conn} do
+      File.mkdir!(@dir <> "/instance/")
+      File.write!(@dir <> "/instance/panel.html", "Custom instance panel")
+
+      conn_resp =
+        conn
+        |> get("/api/pleroma/admin/instance_document/instance-panel")
+
+      assert %{"url" => url} = json_response_and_validate_schema(conn_resp, 200)
+      index = get(build_conn(), url)
+      assert html_response(index, 200) == "Custom instance panel"
+
+      conn
+      |> delete("/api/pleroma/admin/instance_document/instance-panel")
+      |> json_response_and_validate_schema(200)
+
+      conn_resp =
+        conn
+        |> get("/api/pleroma/admin/instance_document/instance-panel")
+
+      assert %{"url" => url} = json_response_and_validate_schema(conn_resp, 200)
+      index = get(build_conn(), url)
+      response = html_response(index, 200)
+      assert String.contains?(response, @default_instance_panel)
+    end
+  end
+end

From f58262c673616661baf9750c216a07d239ae99c3 Mon Sep 17 00:00:00 2001
From: stwf <steven.fuchs@dockyard.com>
Date: Thu, 17 Sep 2020 09:48:17 -0400
Subject: [PATCH 04/12] add description to changelog

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 75357f05e..a7a4f08ae 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated.
 - Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated.
+- The `discoverable` field in the `User` struct will now add a NOINDEX metatag to profile pages when false.
 
 ### Removed
 

From c711a2b15761db9d2d30035e9fee0783f0bf77b0 Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Thu, 17 Sep 2020 16:54:38 +0300
Subject: [PATCH 05/12] Return the file content for `GET
 /api/pleroma/admin/instance_document/:document_name`

---
 docs/API/admin_api.md                            | 12 ++++++------
 .../controllers/instance_document_controller.ex  |  8 ++++++--
 .../admin/instance_document_operation.ex         |  9 ++++++++-
 lib/pleroma/web/instance_document.ex             |  2 +-
 .../instance_document_controller_test.exs        | 16 +++++-----------
 5 files changed, 26 insertions(+), 21 deletions(-)

diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md
index eba92dd1f..7992db58f 100644
--- a/docs/API/admin_api.md
+++ b/docs/API/admin_api.md
@@ -1458,23 +1458,23 @@ Loads json generated from `config/descriptions.exs`.
 
 ## `GET /api/pleroma/admin/instance_document/:document_name`
 
-### Gets an instance document
+### Get an instance document
 
 - Authentication: required
 
 - Response:
 
-``` json
-{
-  "url": "https://example.com/instance/panel.html"
-}
+Returns the content of the document
+
+```html
+<h1>Instance panel</h1>
 ```
 
 ## `PATCH /api/pleroma/admin/instance_document/:document_name`
 - Params:
   - `file` (the file to be uploaded, using multipart form data.)
 
-### Updates an instance document
+### Update an instance document
 
 - Authentication: required
 
diff --git a/lib/pleroma/web/admin_api/controllers/instance_document_controller.ex b/lib/pleroma/web/admin_api/controllers/instance_document_controller.ex
index 2144e44ac..504d9b517 100644
--- a/lib/pleroma/web/admin_api/controllers/instance_document_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/instance_document_controller.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.AdminAPI.InstanceDocumentController do
   use Pleroma.Web, :controller
 
+  alias Pleroma.Plugs.InstanceStatic
   alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.Web.InstanceDocument
 
@@ -18,8 +19,11 @@ defmodule Pleroma.Web.AdminAPI.InstanceDocumentController do
   plug(OAuthScopesPlug, %{scopes: ["write"], admin: true} when action in [:update, :delete])
 
   def show(conn, %{name: document_name}) do
-    with {:ok, url} <- InstanceDocument.get(document_name) do
-      json(conn, %{"url" => url})
+    with {:ok, url} <- InstanceDocument.get(document_name),
+         {:ok, content} <- File.read(InstanceStatic.file_path(url)) do
+      conn
+      |> put_resp_content_type("text/html")
+      |> send_resp(200, content)
     end
   end
 
diff --git a/lib/pleroma/web/api_spec/operations/admin/instance_document_operation.ex b/lib/pleroma/web/api_spec/operations/admin/instance_document_operation.ex
index e0eb993fb..a120ff4e8 100644
--- a/lib/pleroma/web/api_spec/operations/admin/instance_document_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/admin/instance_document_operation.ex
@@ -26,7 +26,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.InstanceDocumentOperation do
         | Helpers.admin_api_params()
       ],
       responses: %{
-        200 => Operation.response("InstanceDocument", "application/json", instance_document()),
+        200 => document_content(),
         400 => Operation.response("Bad Request", "application/json", ApiError),
         403 => Operation.response("Forbidden", "application/json", ApiError),
         404 => Operation.response("Not Found", "application/json", ApiError)
@@ -105,4 +105,11 @@ defmodule Pleroma.Web.ApiSpec.Admin.InstanceDocumentOperation do
       }
     }
   end
+
+  defp document_content do
+    Operation.response("InstanceDocumentContent", "text/html", %Schema{
+      type: :string,
+      example: "<h1>Instance panel</h1>"
+    })
+  end
 end
diff --git a/lib/pleroma/web/instance_document.ex b/lib/pleroma/web/instance_document.ex
index 969a44e41..df5caebf0 100644
--- a/lib/pleroma/web/instance_document.ex
+++ b/lib/pleroma/web/instance_document.ex
@@ -14,7 +14,7 @@ defmodule Pleroma.Web.InstanceDocument do
   @spec get(String.t()) :: {:ok, String.t()} | {:error, atom()}
   def get(document_name) do
     case Map.fetch(@instance_documents, document_name) do
-      {:ok, path} -> {:ok, Path.join(Endpoint.url(), path)}
+      {:ok, path} -> {:ok, path}
       _ -> {:error, :not_found}
     end
   end
diff --git a/test/web/admin_api/controllers/instance_document_controller_test.exs b/test/web/admin_api/controllers/instance_document_controller_test.exs
index 60dcc9dff..5f7b042f6 100644
--- a/test/web/admin_api/controllers/instance_document_controller_test.exs
+++ b/test/web/admin_api/controllers/instance_document_controller_test.exs
@@ -33,10 +33,8 @@ defmodule Pleroma.Web.AdminAPI.InstanceDocumentControllerTest do
     test "return the instance document url", %{conn: conn} do
       conn = get(conn, "/api/pleroma/admin/instance_document/instance-panel")
 
-      assert %{"url" => url} = json_response_and_validate_schema(conn, 200)
-      index = get(build_conn(), url)
-      response = html_response(index, 200)
-      assert String.contains?(response, @default_instance_panel)
+      assert content = html_response(conn, 200)
+      assert String.contains?(content, @default_instance_panel)
     end
 
     test "it returns 403 if requested by a non-admin" do
@@ -91,9 +89,7 @@ defmodule Pleroma.Web.AdminAPI.InstanceDocumentControllerTest do
         conn
         |> get("/api/pleroma/admin/instance_document/instance-panel")
 
-      assert %{"url" => url} = json_response_and_validate_schema(conn_resp, 200)
-      index = get(build_conn(), url)
-      assert html_response(index, 200) == "Custom instance panel"
+      assert html_response(conn_resp, 200) == "Custom instance panel"
 
       conn
       |> delete("/api/pleroma/admin/instance_document/instance-panel")
@@ -103,10 +99,8 @@ defmodule Pleroma.Web.AdminAPI.InstanceDocumentControllerTest do
         conn
         |> get("/api/pleroma/admin/instance_document/instance-panel")
 
-      assert %{"url" => url} = json_response_and_validate_schema(conn_resp, 200)
-      index = get(build_conn(), url)
-      response = html_response(index, 200)
-      assert String.contains?(response, @default_instance_panel)
+      assert content = html_response(conn_resp, 200)
+      assert String.contains?(content, @default_instance_panel)
     end
   end
 end

From f7e40f7ef134a3030aa61114daa39810efb5889d Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Thu, 17 Sep 2020 09:32:50 -0500
Subject: [PATCH 06/12] Deny ConfigDB migration when deprecated settings found

---
 lib/mix/tasks/pleroma/config.ex            | 10 ++++-
 lib/pleroma/config/deprecation_warnings.ex | 43 ++++++++++++++++++----
 test/tasks/config_test.exs                 | 13 +++++++
 3 files changed, 56 insertions(+), 10 deletions(-)

diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex
index 904c5a74b..18f99318d 100644
--- a/lib/mix/tasks/pleroma/config.ex
+++ b/lib/mix/tasks/pleroma/config.ex
@@ -32,7 +32,8 @@ defmodule Mix.Tasks.Pleroma.Config do
 
   @spec migrate_to_db(Path.t() | nil) :: any()
   def migrate_to_db(file_path \\ nil) do
-    if Pleroma.Config.get([:configurable_from_database]) do
+    with true <- Pleroma.Config.get([:configurable_from_database]),
+         :ok <- Pleroma.Config.DeprecationWarnings.warn() do
       config_file =
         if file_path do
           file_path
@@ -46,7 +47,8 @@ defmodule Mix.Tasks.Pleroma.Config do
 
       do_migrate_to_db(config_file)
     else
-      migration_error()
+      :error -> deprecation_error()
+      _ -> migration_error()
     end
   end
 
@@ -120,6 +122,10 @@ defmodule Mix.Tasks.Pleroma.Config do
     )
   end
 
+  defp deprecation_error do
+    shell_error("Migration is not allowed until all deprecation warnings have been resolved.")
+  end
+
   if Code.ensure_loaded?(Config.Reader) do
     defp config_header, do: "import Config\r\n\r\n"
     defp read_file(config_file), do: Config.Reader.read_imports!(config_file)
diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex
index 412d55a77..98c4dc9c8 100644
--- a/lib/pleroma/config/deprecation_warnings.ex
+++ b/lib/pleroma/config/deprecation_warnings.ex
@@ -26,6 +26,10 @@ defmodule Pleroma.Config.DeprecationWarnings do
       !!!DEPRECATION WARNING!!!
       You are using the old configuration mechanism for the hellthread filter. Please check config.md.
       """)
+
+      :error
+    else
+      :ok
     end
   end
 
@@ -47,17 +51,26 @@ defmodule Pleroma.Config.DeprecationWarnings do
 
       config :pleroma, :mrf_user_allowlist, #{inspect(rewritten, pretty: true)}
       """)
+
+      :error
+    else
+      :ok
     end
   end
 
   def warn do
-    check_hellthread_threshold()
-    mrf_user_allowlist()
-    check_old_mrf_config()
-    check_media_proxy_whitelist_config()
-    check_welcome_message_config()
-    check_gun_pool_options()
-    check_activity_expiration_config()
+    with :ok <- check_hellthread_threshold(),
+         :ok <- mrf_user_allowlist(),
+         :ok <- check_old_mrf_config(),
+         :ok <- check_media_proxy_whitelist_config(),
+         :ok <- check_welcome_message_config(),
+         :ok <- check_gun_pool_options(),
+         :ok <- check_activity_expiration_config() do
+      :ok
+    else
+      _ ->
+        :error
+    end
   end
 
   def check_welcome_message_config do
@@ -74,6 +87,10 @@ defmodule Pleroma.Config.DeprecationWarnings do
       \n* `config :pleroma, :instance, welcome_user_nickname` is now `config :pleroma, :welcome, :direct_message, :sender_nickname`
       \n* `config :pleroma, :instance, welcome_message` is now `config :pleroma, :welcome, :direct_message, :message`
       """)
+
+      :error
+    else
+      :ok
     end
   end
 
@@ -101,8 +118,11 @@ defmodule Pleroma.Config.DeprecationWarnings do
           end
       end)
 
-    if warning != "" do
+    if warning == "" do
+      :ok
+    else
       Logger.warn(warning_preface <> warning)
+      :error
     end
   end
 
@@ -115,6 +135,10 @@ defmodule Pleroma.Config.DeprecationWarnings do
       !!!DEPRECATION WARNING!!!
       Your config is using old format (only domain) for MediaProxy whitelist option. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later.
       """)
+
+      :error
+    else
+      :ok
     end
   end
 
@@ -157,6 +181,9 @@ defmodule Pleroma.Config.DeprecationWarnings do
       Logger.warn(Enum.join([warning_preface | pool_warnings]))
 
       Config.put(:pools, updated_config)
+      :error
+    else
+      :ok
     end
   end
 
diff --git a/test/tasks/config_test.exs b/test/tasks/config_test.exs
index fb12e7fb3..f36648829 100644
--- a/test/tasks/config_test.exs
+++ b/test/tasks/config_test.exs
@@ -40,6 +40,19 @@ defmodule Mix.Tasks.Pleroma.ConfigTest do
       on_exit(fn -> Application.put_env(:quack, :level, initial) end)
     end
 
+    @tag capture_log: true
+    test "config migration refused when deprecated settings are found" do
+      clear_config([:media_proxy, :whitelist], ["domain_without_scheme.com"])
+      assert Repo.all(ConfigDB) == []
+
+      Mix.Tasks.Pleroma.Config.migrate_to_db("test/fixtures/config/temp.secret.exs")
+
+      assert_received {:mix_shell, :error, [message]}
+
+      assert message =~
+               "Migration is not allowed until all deprecation warnings have been resolved."
+    end
+
     test "filtered settings are migrated to db" do
       assert Repo.all(ConfigDB) == []
 

From 41939e3175cf31884cb84acd136c303a84c77f8c Mon Sep 17 00:00:00 2001
From: stwf <steven.fuchs@dockyard.com>
Date: Mon, 14 Sep 2020 11:40:52 -0400
Subject: [PATCH 07/12] User search respect discoverable flag

---
 lib/pleroma/user/search.ex                    |  5 +++
 .../tesla_mock/admin@mastdon.example.org.json | 44 +++++++++++--------
 ...ps___osada.macgirvin.com_channel_mike.json |  3 +-
 test/support/factory.ex                       |  1 +
 test/web/admin_api/search_test.exs            |  9 ++++
 .../mastodon_api/views/account_view_test.exs  |  4 +-
 6 files changed, 45 insertions(+), 21 deletions(-)

diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex
index 7babd47ea..b8c648672 100644
--- a/lib/pleroma/user/search.ex
+++ b/lib/pleroma/user/search.ex
@@ -52,6 +52,7 @@ defmodule Pleroma.User.Search do
     |> base_query(following)
     |> filter_blocked_user(for_user)
     |> filter_invisible_users()
+    |> filter_discoverable_users()
     |> filter_internal_users()
     |> filter_blocked_domains(for_user)
     |> fts_search(query_string)
@@ -122,6 +123,10 @@ defmodule Pleroma.User.Search do
     from(q in query, where: q.invisible == false)
   end
 
+  defp filter_discoverable_users(query) do
+    from(q in query, where: q.discoverable == true)
+  end
+
   defp filter_internal_users(query) do
     from(q in query, where: q.actor_type != "Application")
   end
diff --git a/test/fixtures/tesla_mock/admin@mastdon.example.org.json b/test/fixtures/tesla_mock/admin@mastdon.example.org.json
index a911b979a..f961ccb36 100644
--- a/test/fixtures/tesla_mock/admin@mastdon.example.org.json
+++ b/test/fixtures/tesla_mock/admin@mastdon.example.org.json
@@ -1,20 +1,24 @@
 {
-  "@context": ["https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1", {
-    "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
-    "sensitive": "as:sensitive",
-    "movedTo": "as:movedTo",
-    "Hashtag": "as:Hashtag",
-    "ostatus": "http://ostatus.org#",
-    "atomUri": "ostatus:atomUri",
-    "inReplyToAtomUri": "ostatus:inReplyToAtomUri",
-    "conversation": "ostatus:conversation",
-    "toot": "http://joinmastodon.org/ns#",
-    "Emoji": "toot:Emoji",
-    "alsoKnownAs": {
-      "@id": "as:alsoKnownAs",
-      "@type": "@id"
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    "https://w3id.org/security/v1",
+    {
+      "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+      "sensitive": "as:sensitive",
+      "movedTo": "as:movedTo",
+      "Hashtag": "as:Hashtag",
+      "ostatus": "http://ostatus.org#",
+      "atomUri": "ostatus:atomUri",
+      "inReplyToAtomUri": "ostatus:inReplyToAtomUri",
+      "conversation": "ostatus:conversation",
+      "toot": "http://joinmastodon.org/ns#",
+      "Emoji": "toot:Emoji",
+      "alsoKnownAs": {
+        "@id": "as:alsoKnownAs",
+        "@type": "@id"
+      }
     }
-  }],
+  ],
   "id": "http://mastodon.example.org/users/admin",
   "type": "Person",
   "following": "http://mastodon.example.org/users/admin/following",
@@ -23,6 +27,7 @@
   "outbox": "http://mastodon.example.org/users/admin/outbox",
   "preferredUsername": "admin",
   "name": null,
+  "discoverable": "true",
   "summary": "\u003cp\u003e\u003c/p\u003e",
   "url": "http://mastodon.example.org/@admin",
   "manuallyApprovesFollowers": false,
@@ -34,7 +39,8 @@
     "owner": "http://mastodon.example.org/users/admin",
     "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtc4Tir+3ADhSNF6VKrtW\nOU32T01w7V0yshmQei38YyiVwVvFu8XOP6ACchkdxbJ+C9mZud8qWaRJKVbFTMUG\nNX4+6Q+FobyuKrwN7CEwhDALZtaN2IPbaPd6uG1B7QhWorrY+yFa8f2TBM3BxnUy\nI4T+bMIZIEYG7KtljCBoQXuTQmGtuffO0UwJksidg2ffCF5Q+K//JfQagJ3UzrR+\nZXbKMJdAw4bCVJYs4Z5EhHYBwQWiXCyMGTd7BGlmMkY6Av7ZqHKC/owp3/0EWDNz\nNqF09Wcpr3y3e8nA10X40MJqp/wR+1xtxp+YGbq/Cj5hZGBG7etFOmIpVBrDOhry\nBwIDAQAB\n-----END PUBLIC KEY-----\n"
   },
-  "attachment": [{
+  "attachment": [
+    {
       "type": "PropertyValue",
       "name": "foo",
       "value": "bar"
@@ -58,5 +64,7 @@
     "mediaType": "image/png",
     "url": "https://cdn.niu.moe/accounts/headers/000/033/323/original/850b3448fa5fd477.png"
   },
-  "alsoKnownAs": ["http://example.org/users/foo"]
-}
+  "alsoKnownAs": [
+    "http://example.org/users/foo"
+  ]
+}
\ No newline at end of file
diff --git a/test/fixtures/tesla_mock/https___osada.macgirvin.com_channel_mike.json b/test/fixtures/tesla_mock/https___osada.macgirvin.com_channel_mike.json
index c42f3a53c..ca76d6e17 100644
--- a/test/fixtures/tesla_mock/https___osada.macgirvin.com_channel_mike.json
+++ b/test/fixtures/tesla_mock/https___osada.macgirvin.com_channel_mike.json
@@ -8,6 +8,7 @@
     "preferredUsername": "mike",
     "name": "Mike Macgirvin (Osada)",
     "updated": "2018-08-29T03:09:11Z",
+    "discoverable": "true",
     "icon": {
         "type": "Image",
         "mediaType": "image/jpeg",
@@ -51,4 +52,4 @@
         "created": "2018-10-17T07:16:28Z",
         "signatureValue": "WbfFVIPImkd3yNu6brz0CvZaeV242rwAbH0vy8DM4vfnXCxLr5Uv/Wj9gwP+tbooTxGaahAKBeqlGkQp8RLEo37LATrKMRLA/0V6DeeV+C5ORWR9B4WxyWiD3s/9Wf+KesFMtktNLAcMZ5PfnOS/xNYerhnpkp/gWPxtkglmLIWJv+w18A5zZ01JCxsO4QljHbhYaEUPHUfQ97abrkLECeam+FThVwdO6BFCtbjoNXHfzjpSZL/oKyBpi5/fpnqMqOLOQPs5WgBBZJvjEYYkQcoPTyxYI5NGpNbzIjGHPQNuACnOelH16A7L+q4swLWDIaEFeXQ2/5bmqVKZDZZ6usNP4QyTVszwd8jqo27qcDTNibXDUTsTdKpNQvM/3UncBuzuzmUV3FczhtGshIU1/pRVZiQycpVqPlGLvXhP/yZCe+1siyqDd+3uMaS2vkHTObSl5r+VYof+c+TcjrZXHSWnQTg8/X3zkoBWosrQ93VZcwjzMxQoARYv6rphbOoTz7RPmGAXYUt3/PDWkqDlmQDwCpLNNkJo1EidyefZBdD9HXQpCBO0ZU0NHb0JmPvg/+zU0krxlv70bm3RHA/maBETVjroIWzt7EwQEg5pL2hVnvSBG+1wF3BtRVe77etkPOHxLnYYIcAMLlVKCcgDd89DPIziQyruvkx1busHI08="
     }
-}
+}
\ No newline at end of file
diff --git a/test/support/factory.ex b/test/support/factory.ex
index 2fdfabbc5..fb82be0c4 100644
--- a/test/support/factory.ex
+++ b/test/support/factory.ex
@@ -31,6 +31,7 @@ defmodule Pleroma.Factory do
       nickname: sequence(:nickname, &"nick#{&1}"),
       password_hash: Pbkdf2.hash_pwd_salt("test"),
       bio: sequence(:bio, &"Tester Number #{&1}"),
+      discoverable: true,
       last_digest_emailed_at: NaiveDateTime.utc_now(),
       last_refreshed_at: NaiveDateTime.utc_now(),
       notification_settings: %Pleroma.User.NotificationSetting{},
diff --git a/test/web/admin_api/search_test.exs b/test/web/admin_api/search_test.exs
index b974cedd5..d88867c52 100644
--- a/test/web/admin_api/search_test.exs
+++ b/test/web/admin_api/search_test.exs
@@ -177,5 +177,14 @@ defmodule Pleroma.Web.AdminAPI.SearchTest do
       assert total == 3
       assert count == 1
     end
+
+    test "it returns non-discoverable users" do
+      insert(:user)
+      insert(:user, discoverable: false)
+
+      {:ok, _results, total} = Search.user()
+
+      assert total == 2
+    end
   end
 end
diff --git a/test/web/mastodon_api/views/account_view_test.exs b/test/web/mastodon_api/views/account_view_test.exs
index c5f491d6b..a54b765ef 100644
--- a/test/web/mastodon_api/views/account_view_test.exs
+++ b/test/web/mastodon_api/views/account_view_test.exs
@@ -68,7 +68,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do
         sensitive: false,
         pleroma: %{
           actor_type: "Person",
-          discoverable: false
+          discoverable: true
         },
         fields: []
       },
@@ -166,7 +166,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do
         sensitive: false,
         pleroma: %{
           actor_type: "Service",
-          discoverable: false
+          discoverable: true
         },
         fields: []
       },

From dfc621a5291a761f025670153bb58a2005fd0a73 Mon Sep 17 00:00:00 2001
From: stwf <steven.fuchs@dockyard.com>
Date: Thu, 17 Sep 2020 10:13:56 -0400
Subject: [PATCH 08/12] add test and changelog entry

---
 CHANGELOG.md              | 2 +-
 test/user_search_test.exs | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6072d4cbe..5d329fd55 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,8 +10,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated.
 - Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated.
 - The `discoverable` field in the `User` struct will now add a NOINDEX metatag to profile pages when false.
+- Users with the `discoverable` field set to false will not show up in searches.
 - Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option).
-
 ### Removed
 
 - **Breaking:** `Pleroma.Workers.Cron.StatsWorker` setting from Oban `:crontab` (moved to a simpler implementation).
diff --git a/test/user_search_test.exs b/test/user_search_test.exs
index 01976bf58..8529ce6db 100644
--- a/test/user_search_test.exs
+++ b/test/user_search_test.exs
@@ -25,6 +25,14 @@ defmodule Pleroma.UserSearchTest do
       assert found_user.id == user.id
     end
 
+    test "excludes users when discoverable is false" do
+      insert(:user, %{nickname: "john 3000", discoverable: false})
+      insert(:user, %{nickname: "john 3001"})
+
+      users = User.search("john")
+      assert Enum.count(users) == 1
+    end
+
     test "excludes service actors from results" do
       insert(:user, actor_type: "Application", nickname: "user1")
       service = insert(:user, actor_type: "Service", nickname: "user2")

From 9d77f4abf80f75559456cef06da1a0d3b3b4f7e2 Mon Sep 17 00:00:00 2001
From: stwf <steven.fuchs@dockyard.com>
Date: Thu, 17 Sep 2020 12:32:40 -0400
Subject: [PATCH 09/12] adapt to new user factory behavior

---
 test/web/metadata/metadata_test.exs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/web/metadata/metadata_test.exs b/test/web/metadata/metadata_test.exs
index 054844597..ca6cbe67f 100644
--- a/test/web/metadata/metadata_test.exs
+++ b/test/web/metadata/metadata_test.exs
@@ -16,7 +16,7 @@ defmodule Pleroma.Web.MetadataTest do
     end
 
     test "for local user" do
-      user = insert(:user)
+      user = insert(:user, discoverable: false)
 
       assert Pleroma.Web.Metadata.build_tags(%{user: user}) =~
                "<meta content=\"noindex, noarchive\" name=\"robots\">"
@@ -40,7 +40,7 @@ defmodule Pleroma.Web.MetadataTest do
 
     test "search exclusion metadata is included" do
       clear_config([:instance, :public], false)
-      user = insert(:user, bio: "This is my secret fedi account bio")
+      user = insert(:user, bio: "This is my secret fedi account bio", discoverable: false)
 
       assert ~s(<meta content="noindex, noarchive" name="robots">) ==
                Pleroma.Web.Metadata.build_tags(%{user: user})

From 3a0f99ed35a84145e713d4c640c50dc82c1b0dbb Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 14 Sep 2020 13:52:13 +0200
Subject: [PATCH 10/12] KeywordPolicy: Still match when fields are absent

---
 .../web/activity_pub/mrf/keyword_policy.ex    | 61 +++++++++----------
 1 file changed, 29 insertions(+), 32 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex
index 15e09dcf0..db66cfa3e 100644
--- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex
@@ -20,9 +20,17 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
     String.match?(string, pattern)
   end
 
-  defp check_reject(%{"object" => %{"content" => content, "summary" => summary}} = message) do
+  defp object_payload(%{} = object) do
+    [object["content"], object["summary"], object["name"]]
+    |> Enum.filter(& &1)
+    |> Enum.join("\n")
+  end
+
+  defp check_reject(%{"object" => %{} = object} = message) do
+    payload = object_payload(object)
+
     if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern ->
-         string_matches?(content, pattern) or string_matches?(summary, pattern)
+         string_matches?(payload, pattern)
        end) do
       {:reject, "[KeywordPolicy] Matches with rejected keyword"}
     else
@@ -30,12 +38,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
     end
   end
 
-  defp check_ftl_removal(
-         %{"to" => to, "object" => %{"content" => content, "summary" => summary}} = message
-       ) do
+  defp check_ftl_removal(%{"to" => to, "object" => %{} = object} = message) do
+    payload = object_payload(object)
+
     if Pleroma.Constants.as_public() in to and
          Enum.any?(Pleroma.Config.get([:mrf_keyword, :federated_timeline_removal]), fn pattern ->
-           string_matches?(content, pattern) or string_matches?(summary, pattern)
+           string_matches?(payload, pattern)
          end) do
       to = List.delete(to, Pleroma.Constants.as_public())
       cc = [Pleroma.Constants.as_public() | message["cc"] || []]
@@ -51,35 +59,24 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
     end
   end
 
-  defp check_replace(%{"object" => %{"content" => content, "summary" => summary}} = message) do
-    content =
-      if is_binary(content) do
-        content
-      else
-        ""
-      end
+  defp check_replace(%{"object" => %{} = object} = message) do
+    object =
+      ["content", "name", "summary"]
+      |> Enum.filter(fn field -> Map.has_key?(object, field) && object[field] end)
+      |> Enum.reduce(object, fn field, object ->
+        data =
+          Enum.reduce(
+            Pleroma.Config.get([:mrf_keyword, :replace]),
+            object[field],
+            fn {pat, repl}, acc -> String.replace(acc, pat, repl) end
+          )
 
-    summary =
-      if is_binary(summary) do
-        summary
-      else
-        ""
-      end
+        Map.put(object, field, data)
+      end)
 
-    {content, summary} =
-      Enum.reduce(
-        Pleroma.Config.get([:mrf_keyword, :replace]),
-        {content, summary},
-        fn {pattern, replacement}, {content_acc, summary_acc} ->
-          {String.replace(content_acc, pattern, replacement),
-           String.replace(summary_acc, pattern, replacement)}
-        end
-      )
+    message = Map.put(message, "object", object)
 
-    {:ok,
-     message
-     |> put_in(["object", "content"], content)
-     |> put_in(["object", "summary"], summary)}
+    {:ok, message}
   end
 
   @impl true

From abf25e5d5254edc88a65610bf5a0fd7e52f545c3 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sat, 12 Sep 2020 12:05:36 +0200
Subject: [PATCH 11/12] Create MRF.filter_pipeline to inject :object_data when
 present

---
 CHANGELOG.md                                  |  6 +++++
 lib/pleroma/web/activity_pub/mrf.ex           | 24 ++++++++++++++++---
 .../web/activity_pub/mrf/subchain_policy.ex   |  3 +--
 lib/pleroma/web/activity_pub/pipeline.ex      |  8 +++++--
 test/web/activity_pub/pipeline_test.exs       | 16 ++++++-------
 .../controllers/chat_controller_test.exs      | 17 +++++++++++++
 6 files changed, 59 insertions(+), 15 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1c25c60a3..de11dd7a9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -27,6 +27,12 @@ switched to a new configuration mechanism, however it was not officially removed
 - Welcome Chat messages preventing user registration with MRF Simple Policy applied to the local instance
 - Mastodon API: the public timeline returning an error when the `reply_visibility` parameter is set to `self` for an unauthenticated user
 
+## Unreleased-patch
+
+### Security
+
+- Fix most MRF rules either crashing or not being applied to objects passed into the Common Pipeline (ChatMessage, Question, Answer, Audio, Event)
+
 ## [2.1.1] - 2020-09-08
 
 ### Security
diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex
index 206d6af52..5e5361082 100644
--- a/lib/pleroma/web/activity_pub/mrf.ex
+++ b/lib/pleroma/web/activity_pub/mrf.ex
@@ -5,16 +5,34 @@
 defmodule Pleroma.Web.ActivityPub.MRF do
   @callback filter(Map.t()) :: {:ok | :reject, Map.t()}
 
-  def filter(policies, %{} = object) do
+  def filter(policies, %{} = message) do
     policies
-    |> Enum.reduce({:ok, object}, fn
-      policy, {:ok, object} -> policy.filter(object)
+    |> Enum.reduce({:ok, message}, fn
+      policy, {:ok, message} -> policy.filter(message)
       _, error -> error
     end)
   end
 
   def filter(%{} = object), do: get_policies() |> filter(object)
 
+  def pipeline_filter(%{} = message, meta) do
+    object = meta[:object_data]
+    ap_id = message["object"]
+
+    if object && ap_id do
+      with {:ok, message} <- filter(Map.put(message, "object", object)) do
+        meta = Keyword.put(meta, :object_data, message["object"])
+        {:ok, Map.put(message, "object", ap_id), meta}
+      else
+        {err, message} -> {err, message, meta}
+      end
+    else
+      {err, message} = filter(message)
+
+      {err, message, meta}
+    end
+  end
+
   def get_policies do
     Pleroma.Config.get([:mrf, :policies], []) |> get_policies()
   end
diff --git a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex
index c9f20571f..048052da6 100644
--- a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex
@@ -28,8 +28,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SubchainPolicy do
         }"
       )
 
-      subchain
-      |> MRF.filter(message)
+      MRF.filter(subchain, message)
     else
       _e -> {:ok, message}
     end
diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex
index 36e325c37..2db86f116 100644
--- a/lib/pleroma/web/activity_pub/pipeline.ex
+++ b/lib/pleroma/web/activity_pub/pipeline.ex
@@ -26,13 +26,17 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do
 
       {:error, e} ->
         {:error, e}
+
+      {:reject, e} ->
+        {:reject, e}
     end
   end
 
   def do_common_pipeline(object, meta) do
     with {_, {:ok, validated_object, meta}} <-
            {:validate_object, ObjectValidator.validate(object, meta)},
-         {_, {:ok, mrfd_object}} <- {:mrf_object, MRF.filter(validated_object)},
+         {_, {:ok, mrfd_object, meta}} <-
+           {:mrf_object, MRF.pipeline_filter(validated_object, meta)},
          {_, {:ok, activity, meta}} <-
            {:persist_object, ActivityPub.persist(mrfd_object, meta)},
          {_, {:ok, activity, meta}} <-
@@ -40,7 +44,7 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do
          {_, {:ok, _}} <- {:federation, maybe_federate(activity, meta)} do
       {:ok, activity, meta}
     else
-      {:mrf_object, {:reject, _}} -> {:ok, nil, meta}
+      {:mrf_object, {:reject, message, _}} -> {:reject, message}
       e -> {:error, e}
     end
   end
diff --git a/test/web/activity_pub/pipeline_test.exs b/test/web/activity_pub/pipeline_test.exs
index f2a231eaf..210a06563 100644
--- a/test/web/activity_pub/pipeline_test.exs
+++ b/test/web/activity_pub/pipeline_test.exs
@@ -26,7 +26,7 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do
         {
           Pleroma.Web.ActivityPub.MRF,
           [],
-          [filter: fn o -> {:ok, o} end]
+          [pipeline_filter: fn o, m -> {:ok, o, m} end]
         },
         {
           Pleroma.Web.ActivityPub.ActivityPub,
@@ -51,7 +51,7 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do
                  Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
 
         assert_called(Pleroma.Web.ActivityPub.ObjectValidator.validate(activity, meta))
-        assert_called(Pleroma.Web.ActivityPub.MRF.filter(activity))
+        assert_called(Pleroma.Web.ActivityPub.MRF.pipeline_filter(activity, meta))
         assert_called(Pleroma.Web.ActivityPub.ActivityPub.persist(activity, meta))
         assert_called(Pleroma.Web.ActivityPub.SideEffects.handle(activity, meta))
         refute called(Pleroma.Web.Federator.publish(activity))
@@ -68,7 +68,7 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do
         {
           Pleroma.Web.ActivityPub.MRF,
           [],
-          [filter: fn o -> {:ok, o} end]
+          [pipeline_filter: fn o, m -> {:ok, o, m} end]
         },
         {
           Pleroma.Web.ActivityPub.ActivityPub,
@@ -93,7 +93,7 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do
                  Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
 
         assert_called(Pleroma.Web.ActivityPub.ObjectValidator.validate(activity, meta))
-        assert_called(Pleroma.Web.ActivityPub.MRF.filter(activity))
+        assert_called(Pleroma.Web.ActivityPub.MRF.pipeline_filter(activity, meta))
         assert_called(Pleroma.Web.ActivityPub.ActivityPub.persist(activity, meta))
         assert_called(Pleroma.Web.ActivityPub.SideEffects.handle(activity, meta))
         assert_called(Pleroma.Web.Federator.publish(activity))
@@ -109,7 +109,7 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do
         {
           Pleroma.Web.ActivityPub.MRF,
           [],
-          [filter: fn o -> {:ok, o} end]
+          [pipeline_filter: fn o, m -> {:ok, o, m} end]
         },
         {
           Pleroma.Web.ActivityPub.ActivityPub,
@@ -131,7 +131,7 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do
                  Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
 
         assert_called(Pleroma.Web.ActivityPub.ObjectValidator.validate(activity, meta))
-        assert_called(Pleroma.Web.ActivityPub.MRF.filter(activity))
+        assert_called(Pleroma.Web.ActivityPub.MRF.pipeline_filter(activity, meta))
         assert_called(Pleroma.Web.ActivityPub.ActivityPub.persist(activity, meta))
         assert_called(Pleroma.Web.ActivityPub.SideEffects.handle(activity, meta))
       end
@@ -148,7 +148,7 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do
         {
           Pleroma.Web.ActivityPub.MRF,
           [],
-          [filter: fn o -> {:ok, o} end]
+          [pipeline_filter: fn o, m -> {:ok, o, m} end]
         },
         {
           Pleroma.Web.ActivityPub.ActivityPub,
@@ -170,7 +170,7 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do
                  Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
 
         assert_called(Pleroma.Web.ActivityPub.ObjectValidator.validate(activity, meta))
-        assert_called(Pleroma.Web.ActivityPub.MRF.filter(activity))
+        assert_called(Pleroma.Web.ActivityPub.MRF.pipeline_filter(activity, meta))
         assert_called(Pleroma.Web.ActivityPub.ActivityPub.persist(activity, meta))
         assert_called(Pleroma.Web.ActivityPub.SideEffects.handle(activity, meta))
       end
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index 7be5fe09c..32c23e9d7 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -126,6 +126,23 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
 
       assert result["attachment"]
     end
+
+    test "gets MRF reason when rejected", %{conn: conn, user: user} do
+      clear_config([:mrf_keyword, :reject], ["GNO"])
+      clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.KeywordPolicy])
+
+      other_user = insert(:user)
+
+      {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
+
+      result =
+        conn
+        |> put_req_header("content-type", "application/json")
+        |> post("/api/v1/pleroma/chats/#{chat.id}/messages", %{"content" => "GNO/Linux"})
+        |> json_response_and_validate_schema(200)
+
+      assert result == %{}
+    end
   end
 
   describe "DELETE /api/v1/pleroma/chats/:id/messages/:message_id" do

From 7bf269fe836ded974d2187c6b36eba4ab185ff25 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 14 Sep 2020 14:07:22 +0200
Subject: [PATCH 12/12] Fix MRF reject for ChatMessage

---
 lib/pleroma/web/api_spec/operations/chat_operation.ex |  3 ++-
 .../web/api_spec/operations/status_operation.ex       |  2 +-
 lib/pleroma/web/common_api/common_api.ex              |  3 +++
 .../web/pleroma_api/controllers/chat_controller.ex    | 10 ++++++++++
 test/web/common_api/common_api_test.exs               | 11 +++++++++++
 .../pleroma_api/controllers/chat_controller_test.exs  |  6 +++---
 6 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index b1a0d26ab..56554d5b4 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -184,7 +184,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do
             "application/json",
             ChatMessage
           ),
-        400 => Operation.response("Bad Request", "application/json", ApiError)
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        422 => Operation.response("MRF Rejection", "application/json", ApiError)
       },
       security: [
         %{
diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex
index 5bd4619d5..d7ebde6f6 100644
--- a/lib/pleroma/web/api_spec/operations/status_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/status_operation.ex
@@ -55,7 +55,7 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do
             "application/json",
             %Schema{oneOf: [Status, ScheduledStatus]}
           ),
-        422 => Operation.response("Bad Request", "application/json", ApiError)
+        422 => Operation.response("Bad Request / MRF Rejection", "application/json", ApiError)
       }
     }
   end
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index a8c83bc8f..60a50b027 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -48,6 +48,9 @@ defmodule Pleroma.Web.CommonAPI do
               local: true
             )} do
       {:ok, activity}
+    else
+      {:common_pipeline, {:reject, _} = e} -> e
+      e -> e
     end
   end
 
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 27c9a2e0f..867cff829 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -90,6 +90,16 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
       conn
       |> put_view(MessageReferenceView)
       |> render("show.json", chat_message_reference: cm_ref)
+    else
+      {:reject, message} ->
+        conn
+        |> put_status(:unprocessable_entity)
+        |> json(%{error: message})
+
+      {:error, message} ->
+        conn
+        |> put_status(:bad_request)
+        |> json(%{error: message})
     end
   end
 
diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs
index f5559f932..2eab64e8b 100644
--- a/test/web/common_api/common_api_test.exs
+++ b/test/web/common_api/common_api_test.exs
@@ -217,6 +217,17 @@ defmodule Pleroma.Web.CommonAPITest do
 
       assert message == :content_too_long
     end
+
+    test "it reject messages via MRF" do
+      clear_config([:mrf_keyword, :reject], ["GNO"])
+      clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.KeywordPolicy])
+
+      author = insert(:user)
+      recipient = insert(:user)
+
+      assert {:reject, "[KeywordPolicy] Matches with rejected keyword"} ==
+               CommonAPI.post_chat_message(author, recipient, "GNO/Linux")
+    end
   end
 
   describe "unblocking" do
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index 32c23e9d7..44a78a738 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -100,7 +100,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
         |> post("/api/v1/pleroma/chats/#{chat.id}/messages")
         |> json_response_and_validate_schema(400)
 
-      assert result
+      assert %{"error" => "no_content"} == result
     end
 
     test "it works with an attachment", %{conn: conn, user: user} do
@@ -139,9 +139,9 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
         conn
         |> put_req_header("content-type", "application/json")
         |> post("/api/v1/pleroma/chats/#{chat.id}/messages", %{"content" => "GNO/Linux"})
-        |> json_response_and_validate_schema(200)
+        |> json_response_and_validate_schema(422)
 
-      assert result == %{}
+      assert %{"error" => "[KeywordPolicy] Matches with rejected keyword"} == result
     end
   end