Merge branch 'develop' of https://git.pleroma.social/pleroma/pleroma into develop

2019-12-23 19:07:20 +00:00 · 2019-12-23 19:07:20 +00:00 · 5f4fc0c373
commit 5f4fc0c373
parent 5fb1c96500 10ce7185c2
24 changed files with 167 additions and 100 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Deprecated `User.Info` embedded schema (fields moved to `User`)
 - Store status data inside Flag activity
 - Deprecated (reorganized as `UserRelationship` entity) User fields with user AP IDs (`blocks`, `mutes`, `muted_reblogs`, `muted_notifications`, `subscribers`).
+- Logger: default log level changed from `warn` to `info`.
 <details>
  <summary>API Changes</summary>

@ -51,6 +52,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Support for `X-Forwarded-For` and similar HTTP headers which used by reverse proxies to pass a real user IP address to the backend. Must not be enabled unless your instance is behind at least one reverse proxy (such as Nginx, Apache HTTPD or Varnish Cache).
 - MRF: New module which handles incoming posts based on their age. By default, all incoming posts that are older than 2 days will be unlisted and not shown to their followers.
 - User notification settings: Add `privacy_option` option.
+- Support for custom Elixir modules (such as MRF policies)
 - User settings: Add _This account is a_ option.
 - OAuth: admin scopes support (relevant setting: `[:auth, :enforce_oauth_admin_scope_usage]`).
 <details>
--- a/config/config.exs
+++ b/config/config.exs
@ -67,7 +67,7 @@ config :pleroma, Pleroma.Scheduler,

 config :pleroma, Pleroma.Captcha,
  enabled: true,
-  seconds_valid: 60,
+  seconds_valid: 3000,
  method: Pleroma.Captcha.Native

 config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch"
@ -621,6 +621,8 @@ config :pleroma, :web_cache_ttl,
  activity_pub: nil,
  activity_pub_question: 30_000

+config :pleroma, :modules, runtime_dir: "instance/modules"
+
 config :swarm, node_blacklist: [~r/myhtml_.*$/]
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
--- a/config/prod.exs
+++ b/config/prod.exs
@ -20,8 +20,8 @@ config :pleroma, Pleroma.Web.Endpoint,
 config :phoenix, serve_endpoints: true

 # Do not print debug messages in production
-config :logger, :console, level: :warn
-config :logger, :ex_syslogger, level: :warn
+config :logger, :console, level: :info
+config :logger, :ex_syslogger, level: :info

 # ## SSL Support
 #
--- a/config/releases.exs
+++ b/config/releases.exs
@ -2,6 +2,7 @@ import Config

 config :pleroma, :instance, static_dir: "/var/lib/pleroma/static"
 config :pleroma, Pleroma.Uploaders.Local, uploads: "/var/lib/pleroma/uploads"
+config :pleroma, :modules, runtime_dir: "/var/lib/pleroma/modules"

 config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs"

--- a/config/test.exs
+++ b/config/test.exs
@ -95,6 +95,8 @@ config :joken, default_signer: "yU8uHKq+yyAkZ11Hx//jcdacWc8yQ1bxAAGrplzB0Zwwjkp3

 config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock

+config :pleroma, :modules, runtime_dir: "test/fixtures/modules"
+
 if File.exists?("./config/test.secret.exs") do
  import_config "test.secret.exs"
 else
--- a/docs/API/pleroma_api.md
+++ b/docs/API/pleroma_api.md
@ -70,59 +70,6 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 * Response: JSON. Returns `{"status": "success"}` if the account was successfully disabled, `{"error": "[error message]"}` otherwise
 * Example response: `{"error": "Invalid password."}`

-## `/api/account/register`
-### Register a new user
-* Method `POST`
-* Authentication: not required
-* Params:
-    * `nickname`
-    * `fullname`
-    * `bio`
-    * `email`
-    * `password`
-    * `confirm`
-    * `captcha_solution`: optional, contains provider-specific captcha solution,
-    * `captcha_token`: optional, contains provider-specific captcha token
-    * `token`: invite token required when the registrations aren't public.
-* Response: JSON. Returns a user object on success, otherwise returns `{"error": "error_msg"}`
-* Example response:
-```json
-{
-	"background_image": null,
-	"cover_photo": "https://pleroma.soykaf.com/images/banner.png",
-	"created_at": "Tue Dec 18 16:55:56 +0000 2018",
-	"default_scope": "public",
-	"description": "blushy-crushy fediverse idol + pleroma dev\nlet's be friends \nぷれろまの生徒会長。謎の外人。日本語OK. \n公主病.",
-	"description_html": "blushy-crushy fediverse idol + pleroma dev.<br />let's be friends <br />ぷれろまの生徒会長。謎の外人。日本語OK. <br />公主病.",
-	"favourites_count": 0,
-	"fields": [],
-	"followers_count": 0,
-	"following": false,
-	"follows_you": false,
-	"friends_count": 0,
-	"id": 6,
-	"is_local": true,
-	"locked": false,
-	"name": "lain",
-	"name_html": "lain",
-	"no_rich_text": false,
-	"pleroma": {
-		"tags": []
-	},
-	"profile_image_url": "https://pleroma.soykaf.com/images/avi.png",
-	"profile_image_url_https": "https://pleroma.soykaf.com/images/avi.png",
-	"profile_image_url_original": "https://pleroma.soykaf.com/images/avi.png",
-	"profile_image_url_profile_size": "https://pleroma.soykaf.com/images/avi.png",
-	"rights": {
-		"delete_others_notice": false
-	},
-	"screen_name": "lain",
-	"statuses_count": 0,
-	"statusnet_blocking": false,
-	"statusnet_profile_url": "https://pleroma.soykaf.com/users/lain"
-}
-```
-
 ## `/api/pleroma/admin/`…
 See [Admin-API](admin_api.md)

--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@ -836,3 +836,7 @@ config :auto_linker,
    rel: "ugc"
  ]
 ```
+
+## Custom Runtime Modules (`:modules`)
+
+* `runtime_dir`: A path to custom Elixir modules (such as MRF policies).
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@ -5,6 +5,7 @@
 defmodule Pleroma.Application do
  import Cachex.Spec
  use Application
+  require Logger

  @name Mix.Project.config()[:name]
  @version Mix.Project.config()[:version]
@ -33,6 +34,7 @@ defmodule Pleroma.Application do
    Pleroma.HTML.compile_scrubbers()
    Pleroma.Config.DeprecationWarnings.warn()
    setup_instrumenters()
+    load_custom_modules()

    # Define workers and child supervisors to be supervised
    children =
@ -68,6 +70,28 @@ defmodule Pleroma.Application do
    Supervisor.start_link(children, opts)
  end

+  def load_custom_modules do
+    dir = Pleroma.Config.get([:modules, :runtime_dir])
+
+    if dir && File.exists?(dir) do
+      dir
+      |> Pleroma.Utils.compile_dir()
+      |> case do
+        {:error, _errors, _warnings} ->
+          raise "Invalid custom modules"
+
+        {:ok, modules, _warnings} ->
+          if @env != :test do
+            Enum.each(modules, fn mod ->
+              Logger.info("Custom module loaded: #{inspect(mod)}")
+            end)
+          end
+
+          :ok
+      end
+    end
+  end
+
  defp setup_instrumenters do
    require Prometheus.Registry

--- a/lib/pleroma/html.ex
+++ b/lib/pleroma/html.ex
@ -10,9 +10,7 @@ defmodule Pleroma.HTML do
    dir = Path.join(:code.priv_dir(:pleroma), "scrubbers")

    dir
-    |> File.ls!()
-    |> Enum.map(&Path.join(dir, &1))
-    |> Kernel.ParallelCompiler.compile()
+    |> Pleroma.Utils.compile_dir()
    |> case do
      {:error, _errors, _warnings} ->
        raise "Compiling scrubbers failed"
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@ -154,7 +154,7 @@ defmodule Pleroma.Object.Fetcher do
  end

  def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
-    Logger.info("Fetching object #{id} via AP")
+    Logger.debug("Fetching object #{id} via AP")

    date = Pleroma.Signature.signed_date()

--- a/lib/pleroma/utils.ex
+++ b/lib/pleroma/utils.ex
@ -0,0 +1,12 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Utils do
+  def compile_dir(dir) when is_binary(dir) do
+    dir
+    |> File.ls!()
+    |> Enum.map(&Path.join(dir, &1))
+    |> Kernel.ParallelCompiler.compile()
+  end
+end
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@ -1298,28 +1298,26 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
  def fetch_follow_information_for_user(user) do
    with {:ok, following_data} <-
           Fetcher.fetch_and_contain_remote_object_from_id(user.following_address),
-         following_count when is_integer(following_count) <- following_data["totalItems"],
         {:ok, hide_follows} <- collection_private(following_data),
         {:ok, followers_data} <-
           Fetcher.fetch_and_contain_remote_object_from_id(user.follower_address),
-         followers_count when is_integer(followers_count) <- followers_data["totalItems"],
         {:ok, hide_followers} <- collection_private(followers_data) do
      {:ok,
       %{
         hide_follows: hide_follows,
-         follower_count: followers_count,
-         following_count: following_count,
+         follower_count: normalize_counter(followers_data["totalItems"]),
+         following_count: normalize_counter(following_data["totalItems"]),
         hide_followers: hide_followers
       }}
    else
-      {:error, _} = e ->
-        e
-
-      e ->
-        {:error, e}
+      {:error, _} = e -> e
+      e -> {:error, e}
    end
  end

+  defp normalize_counter(counter) when is_integer(counter), do: counter
+  defp normalize_counter(_), do: 0
+
  defp maybe_update_follow_information(data) do
    with {:enabled, true} <-
           {:enabled, Pleroma.Config.get([:instance, :external_user_synchronization])},
@ -1339,24 +1337,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
    end
  end

+  defp collection_private(%{"first" => %{"type" => type}})
+       when type in ["CollectionPage", "OrderedCollectionPage"],
+       do: {:ok, false}
+
  defp collection_private(%{"first" => first}) do
-    if is_map(first) and
-         first["type"] in ["CollectionPage", "OrderedCollectionPage"] do
+    with {:ok, %{"type" => type}} when type in ["CollectionPage", "OrderedCollectionPage"] <-
+           Fetcher.fetch_and_contain_remote_object_from_id(first) do
      {:ok, false}
    else
-      with {:ok, %{"type" => type}} when type in ["CollectionPage", "OrderedCollectionPage"] <-
-             Fetcher.fetch_and_contain_remote_object_from_id(first) do
-        {:ok, false}
-      else
-        {:error, {:ok, %{status: code}}} when code in [401, 403] ->
-          {:ok, true}
-
-        {:error, _} = e ->
-          e
-
-        e ->
-          {:error, e}
-      end
+      {:error, {:ok, %{status: code}}} when code in [401, 403] -> {:ok, true}
+      {:error, _} = e -> e
+      e -> {:error, e}
    end
  end

--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@ -257,7 +257,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do

  # only accept relayed Creates
  def inbox(conn, %{"type" => "Create"} = params) do
-    Logger.info(
+    Logger.debug(
      "Signature missing or not from author, relayed Create message, fetching object from source"
    )

@ -270,11 +270,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
    headers = Enum.into(conn.req_headers, %{})

    if String.contains?(headers["signature"], params["actor"]) do
-      Logger.info(
+      Logger.debug(
        "Signature validation error for: #{params["actor"]}, make sure you are forwarding the HTTP Host header!"
      )

-      Logger.info(inspect(conn.req_headers))
+      Logger.debug(inspect(conn.req_headers))
    end

    json(conn, dgettext("errors", "error"))
--- a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex
@ -9,7 +9,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.DropPolicy do

  @impl true
  def filter(object) do
-    Logger.info("REJECTING #{inspect(object)}")
+    Logger.debug("REJECTING #{inspect(object)}")
    {:reject, object}
  end

--- a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex
@ -18,7 +18,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
  ]

  def perform(:prefetch, url) do
-    Logger.info("Prefetching #{inspect(url)}")
+    Logger.debug("Prefetching #{inspect(url)}")

    url
    |> MediaProxy.url()
--- a/lib/pleroma/web/activity_pub/publisher.ex
+++ b/lib/pleroma/web/activity_pub/publisher.ex
@ -48,7 +48,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
  * `id`: the ActivityStreams URI of the message
  """
  def publish_one(%{inbox: inbox, json: json, actor: %User{} = actor, id: id} = params) do
-    Logger.info("Federating #{id} to #{inbox}")
+    Logger.debug("Federating #{id} to #{inbox}")
    %{host: host, path: path} = URI.parse(inbox)

    digest = "SHA-256=" <> (:crypto.hash(:sha256, json) |> Base.encode64())
@ -228,7 +228,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
    public = is_public?(activity)

    if public && Config.get([:instance, :allow_relay]) do
-      Logger.info(fn -> "Relaying #{activity.data["id"]} out" end)
+      Logger.debug(fn -> "Relaying #{activity.data["id"]} out" end)
      Relay.publish(activity)
    end

--- a/lib/pleroma/web/endpoint.ex
+++ b/lib/pleroma/web/endpoint.ex
@ -59,7 +59,7 @@ defmodule Pleroma.Web.Endpoint do

  plug(Pleroma.Plugs.TrailingFormatPlug)
  plug(Plug.RequestId)
-  plug(Plug.Logger)
+  plug(Plug.Logger, log: :debug)

  plug(Pleroma.Plugs.Parsers)

--- a/lib/pleroma/web/federator/federator.ex
+++ b/lib/pleroma/web/federator/federator.ex
@ -58,7 +58,7 @@ defmodule Pleroma.Web.Federator do
  end

  def perform(:incoming_ap_doc, params) do
-    Logger.info("Handling incoming AP activity")
+    Logger.debug("Handling incoming AP activity")

    params = Utils.normalize_params(params)

@ -71,13 +71,13 @@ defmodule Pleroma.Web.Federator do
      {:ok, activity}
    else
      %Activity{} ->
-        Logger.info("Already had #{params["id"]}")
+        Logger.debug("Already had #{params["id"]}")
        :error

      _e ->
        # Just drop those for now
-        Logger.info("Unhandled activity")
-        Logger.info(Jason.encode!(params, pretty: true))
+        Logger.debug("Unhandled activity")
+        Logger.debug(Jason.encode!(params, pretty: true))
        :error
    end
  end
--- a/lib/pleroma/web/federator/publisher.ex
+++ b/lib/pleroma/web/federator/publisher.ex
@ -47,7 +47,7 @@ defmodule Pleroma.Web.Federator.Publisher do
    Config.get([:instance, :federation_publisher_modules])
    |> Enum.each(fn module ->
      if module.is_representable?(activity) do
-        Logger.info("Publishing #{activity.data["id"]} using #{inspect(module)}")
+        Logger.debug("Publishing #{activity.data["id"]} using #{inspect(module)}")
        module.publish(user, activity)
      end
    end)
--- a/lib/pleroma/web/metadata/twitter_card.ex
+++ b/lib/pleroma/web/metadata/twitter_card.ex
@ -31,7 +31,7 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCard do
      if attachments == [] or Metadata.activity_nsfw?(object) do
        [
          image_tag(user),
-          {:meta, [property: "twitter:card", content: "summary_large_image"], []}
+          {:meta, [property: "twitter:card", content: "summary"], []}
        ]
      else
        attachments
--- a/test/fixtures/modules/runtime_module.ex
+++ b/test/fixtures/modules/runtime_module.ex
@ -0,0 +1,9 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule RuntimeModule do
+  @moduledoc """
+  This is a dummy module to test custom runtime modules.
+  """
+end
--- a/test/runtime_test.exs
+++ b/test/runtime_test.exs
@ -0,0 +1,11 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.RuntimeTest do
+  use ExUnit.Case, async: true
+
+  test "it loads custom runtime modules" do
+    assert Code.ensure_compiled?(RuntimeModule)
+  end
+end
--- a/test/web/activity_pub/activity_pub_test.exs
+++ b/test/web/activity_pub/activity_pub_test.exs
@ -1623,6 +1623,44 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
      assert follow_info.following_count == 32
      assert follow_info.hide_follows == true
    end
+
+    test "doesn't crash when follower and following counters are hidden" do
+      mock(fn env ->
+        case env.url do
+          "http://localhost:4001/users/masto_hidden_counters/following" ->
+            json(%{
+              "@context" => "https://www.w3.org/ns/activitystreams",
+              "id" => "http://localhost:4001/users/masto_hidden_counters/followers"
+            })
+
+          "http://localhost:4001/users/masto_hidden_counters/following?page=1" ->
+            %Tesla.Env{status: 403, body: ""}
+
+          "http://localhost:4001/users/masto_hidden_counters/followers" ->
+            json(%{
+              "@context" => "https://www.w3.org/ns/activitystreams",
+              "id" => "http://localhost:4001/users/masto_hidden_counters/following"
+            })
+
+          "http://localhost:4001/users/masto_hidden_counters/followers?page=1" ->
+            %Tesla.Env{status: 403, body: ""}
+        end
+      end)
+
+      user =
+        insert(:user,
+          local: false,
+          follower_address: "http://localhost:4001/users/masto_hidden_counters/followers",
+          following_address: "http://localhost:4001/users/masto_hidden_counters/following"
+        )
+
+      {:ok, follow_info} = ActivityPub.fetch_follow_information_for_user(user)
+
+      assert follow_info.hide_followers == true
+      assert follow_info.follower_count == 0
+      assert follow_info.hide_follows == true
+      assert follow_info.following_count == 0
+    end
  end

  describe "fetch_favourites/3" do
--- a/test/web/metadata/twitter_card_test.exs
+++ b/test/web/metadata/twitter_card_test.exs
@ -26,7 +26,32 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCardTest do
           ]
  end

-  test "it does not render attachments if post is nsfw" do
+  test "it uses summary twittercard if post has no attachment" do
+    user = insert(:user, name: "Jimmy Hendriks", bio: "born 19 March 1994")
+    {:ok, activity} = CommonAPI.post(user, %{"status" => "HI"})
+
+    note =
+      insert(:note, %{
+        data: %{
+          "actor" => user.ap_id,
+          "tag" => [],
+          "id" => "https://pleroma.gov/objects/whatever",
+          "content" => "pleroma in a nutshell"
+        }
+      })
+
+    result = TwitterCard.build_tags(%{object: note, user: user, activity_id: activity.id})
+
+    assert [
+             {:meta, [property: "twitter:title", content: Utils.user_name_string(user)], []},
+             {:meta, [property: "twitter:description", content: "“pleroma in a nutshell”"], []},
+             {:meta, [property: "twitter:image", content: "http://localhost:4001/images/avi.png"],
+              []},
+             {:meta, [property: "twitter:card", content: "summary"], []}
+           ] == result
+  end
+
+  test "it renders avatar not attachment if post is nsfw and unfurl_nsfw is disabled" do
    Pleroma.Config.put([Pleroma.Web.Metadata, :unfurl_nsfw], false)
    user = insert(:user, name: "Jimmy Hendriks", bio: "born 19 March 1994")
    {:ok, activity} = CommonAPI.post(user, %{"status" => "HI"})
@ -67,7 +92,7 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCardTest do
             {:meta, [property: "twitter:description", content: "“pleroma in a nutshell”"], []},
             {:meta, [property: "twitter:image", content: "http://localhost:4001/images/avi.png"],
              []},
-             {:meta, [property: "twitter:card", content: "summary_large_image"], []}
+             {:meta, [property: "twitter:card", content: "summary"], []}
           ] == result
  end