From aedbbec88aa0a9a38e588eabfbecb8058652002b Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sat, 11 Jul 2020 15:48:45 +0300 Subject: [PATCH 01/32] Add Pleroma.Utils.command_available?/1 and use where appropriate --- lib/pleroma/upload/filter/exiftool.ex | 9 ++++++++- lib/pleroma/utils.ex | 15 +++++++++++++++ mix.exs | 6 +++--- test/upload/filter/exiftool_test.exs | 2 ++ 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/upload/filter/exiftool.ex b/lib/pleroma/upload/filter/exiftool.ex index c7fb6aefa..94622acd0 100644 --- a/lib/pleroma/upload/filter/exiftool.ex +++ b/lib/pleroma/upload/filter/exiftool.ex @@ -9,8 +9,15 @@ defmodule Pleroma.Upload.Filter.Exiftool do """ @behaviour Pleroma.Upload.Filter + require Logger + def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do - System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) + if Pleroma.Utils.command_available?("exiftool") do + System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) + else + Logger.warn("exiftool is not available, filter #{__MODULE__} skipped") + end + :ok end diff --git a/lib/pleroma/utils.ex b/lib/pleroma/utils.ex index 6b8e3accf..21d1159be 100644 --- a/lib/pleroma/utils.ex +++ b/lib/pleroma/utils.ex @@ -9,4 +9,19 @@ defmodule Pleroma.Utils do |> Enum.map(&Path.join(dir, &1)) |> Kernel.ParallelCompiler.compile() end + + @doc """ + POSIX-compliant check if command is available in the system + + ## Examples + iex> command_available?("git") + true + iex> command_available?("wrongcmd") + false + + """ + @spec command_available?(String.t()) :: boolean() + def command_available?(command) do + match?({_output, 0}, System.cmd("sh", ["-c", "command -v #{command}"])) + end end diff --git a/mix.exs b/mix.exs index d7992ee37..5cd06e8fd 100644 --- a/mix.exs +++ b/mix.exs @@ -234,10 +234,10 @@ defmodule Pleroma.Mixfile do defp version(version) do identifier_filter = ~r/[^0-9a-z\-]+/i - {_cmdgit, cmdgit_err} = System.cmd("sh", ["-c", "command -v git"]) + git_available? = Pleroma.Utils.command_available?("git") git_pre_release = - if cmdgit_err == 0 do + if git_available? do {tag, tag_err} = System.cmd("git", ["describe", "--tags", "--abbrev=0"], stderr_to_stdout: true) @@ -263,7 +263,7 @@ defmodule Pleroma.Mixfile do # Branch name as pre-release version component, denoted with a dot branch_name = - with 0 <- cmdgit_err, + with true <- git_available?, {branch_name, 0} <- System.cmd("git", ["rev-parse", "--abbrev-ref", "HEAD"]), branch_name <- String.trim(branch_name), branch_name <- System.get_env("PLEROMA_BUILD_BRANCH") || branch_name, diff --git a/test/upload/filter/exiftool_test.exs b/test/upload/filter/exiftool_test.exs index a1b7e46cd..8ed7d650b 100644 --- a/test/upload/filter/exiftool_test.exs +++ b/test/upload/filter/exiftool_test.exs @@ -7,6 +7,8 @@ defmodule Pleroma.Upload.Filter.ExiftoolTest do alias Pleroma.Upload.Filter test "apply exiftool filter" do + assert Pleroma.Utils.command_available?("exiftool") + File.cp!( "test/fixtures/DSCN0010.jpg", "test/fixtures/DSCN0010_tmp.jpg" From 05187d497da1844005eaf96fbcab65840a578bb1 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sat, 11 Jul 2020 16:09:46 +0300 Subject: [PATCH 02/32] One can not simply call application modules from mix.exs --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 5cd06e8fd..a775e54a4 100644 --- a/mix.exs +++ b/mix.exs @@ -234,7 +234,7 @@ defmodule Pleroma.Mixfile do defp version(version) do identifier_filter = ~r/[^0-9a-z\-]+/i - git_available? = Pleroma.Utils.command_available?("git") + git_available? = match?({_output, 0}, System.cmd("sh", ["-c", "command -v git"])) git_pre_release = if git_available? do From 45bd64e2a7a08377e260e93c8e1744166bfc133a Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sat, 11 Jul 2020 18:11:23 +0300 Subject: [PATCH 03/32] Error in Filter.Exiftool if exiftool not found --- lib/pleroma/upload/filter/exiftool.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/upload/filter/exiftool.ex b/lib/pleroma/upload/filter/exiftool.ex index 94622acd0..6a40e152f 100644 --- a/lib/pleroma/upload/filter/exiftool.ex +++ b/lib/pleroma/upload/filter/exiftool.ex @@ -14,11 +14,10 @@ defmodule Pleroma.Upload.Filter.Exiftool do def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do if Pleroma.Utils.command_available?("exiftool") do System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) + :ok else - Logger.warn("exiftool is not available, filter #{__MODULE__} skipped") + {:error, "exiftool command not found"} end - - :ok end def filter(_), do: :ok From 523f1b93a48d88ef8aa04ca17d51d1d0916b6093 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sat, 11 Jul 2020 18:15:51 +0300 Subject: [PATCH 04/32] Remove Logger requirement --- lib/pleroma/upload/filter/exiftool.ex | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/pleroma/upload/filter/exiftool.ex b/lib/pleroma/upload/filter/exiftool.ex index 6a40e152f..e1b976c98 100644 --- a/lib/pleroma/upload/filter/exiftool.ex +++ b/lib/pleroma/upload/filter/exiftool.ex @@ -9,8 +9,6 @@ defmodule Pleroma.Upload.Filter.Exiftool do """ @behaviour Pleroma.Upload.Filter - require Logger - def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do if Pleroma.Utils.command_available?("exiftool") do System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) From 3116a75e80144dff79232c8676bd28ed285a14d9 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sat, 11 Jul 2020 18:22:03 +0300 Subject: [PATCH 05/32] Check if mogrify available before calling it --- lib/pleroma/upload/filter/mogrifun.ex | 9 ++++++--- lib/pleroma/upload/filter/mogrify.ex | 10 +++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/upload/filter/mogrifun.ex b/lib/pleroma/upload/filter/mogrifun.ex index 7d95577a4..8f362333d 100644 --- a/lib/pleroma/upload/filter/mogrifun.ex +++ b/lib/pleroma/upload/filter/mogrifun.ex @@ -35,9 +35,12 @@ defmodule Pleroma.Upload.Filter.Mogrifun do ] def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do - Filter.Mogrify.do_filter(file, [Enum.random(@filters)]) - - :ok + if Pleroma.Utils.command_available?("mogrify") do + Filter.Mogrify.do_filter(file, [Enum.random(@filters)]) + :ok + else + {:error, "mogrify command not found"} + end end def filter(_), do: :ok diff --git a/lib/pleroma/upload/filter/mogrify.ex b/lib/pleroma/upload/filter/mogrify.ex index 2eb758006..4bd0c2eb4 100644 --- a/lib/pleroma/upload/filter/mogrify.ex +++ b/lib/pleroma/upload/filter/mogrify.ex @@ -9,10 +9,14 @@ defmodule Pleroma.Upload.Filter.Mogrify do @type conversions :: conversion() | [conversion()] def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do - filters = Pleroma.Config.get!([__MODULE__, :args]) + if Pleroma.Utils.command_available?("mogrify") do + filters = Pleroma.Config.get!([__MODULE__, :args]) - do_filter(file, filters) - :ok + do_filter(file, filters) + :ok + else + {:error, "mogrify command not found"} + end end def filter(_), do: :ok From c2c3dd46133499db4102a946f07be87efdf82f1a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 2 Aug 2020 13:42:23 -0500 Subject: [PATCH 06/32] Migrate legacy tags set by AdminFE to match TagPolicy, #2010 --- .../20200802170532_fix_legacy_tags.exs | 37 +++++++++++++++++++ .../20200802170532_fix_legacy_tags_test.exs | 24 ++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 priv/repo/migrations/20200802170532_fix_legacy_tags.exs create mode 100644 test/migrations/20200802170532_fix_legacy_tags_test.exs diff --git a/priv/repo/migrations/20200802170532_fix_legacy_tags.exs b/priv/repo/migrations/20200802170532_fix_legacy_tags.exs new file mode 100644 index 000000000..f7274b44e --- /dev/null +++ b/priv/repo/migrations/20200802170532_fix_legacy_tags.exs @@ -0,0 +1,37 @@ +# Fix legacy tags set by AdminFE that don't align with TagPolicy MRF + +defmodule Pleroma.Repo.Migrations.FixLegacyTags do + use Ecto.Migration + alias Pleroma.Repo + alias Pleroma.User + import Ecto.Query + + @old_new_map %{ + "force_nsfw" => "mrf_tag:media-force-nsfw", + "strip_media" => "mrf_tag:media-strip", + "force_unlisted" => "mrf_tag:force-unlisted", + "sandbox" => "mrf_tag:sandbox", + "disable_remote_subscription" => "mrf_tag:disable-remote-subscription", + "disable_any_subscription" => "mrf_tag:disable-any-subscription" + } + + def change do + legacy_tags = Map.keys(@old_new_map) + + from(u in User, where: fragment("? && ?", u.tags, ^legacy_tags)) + |> Repo.all() + |> Enum.each(fn user -> + fix_tags_changeset(user) + |> Repo.update() + end) + end + + defp fix_tags_changeset(%User{tags: tags} = user) do + new_tags = + Enum.map(tags, fn tag -> + Map.get(@old_new_map, tag, tag) + end) + + Ecto.Changeset.change(user, tags: new_tags) + end +end diff --git a/test/migrations/20200802170532_fix_legacy_tags_test.exs b/test/migrations/20200802170532_fix_legacy_tags_test.exs new file mode 100644 index 000000000..3b4dee407 --- /dev/null +++ b/test/migrations/20200802170532_fix_legacy_tags_test.exs @@ -0,0 +1,24 @@ +defmodule Pleroma.Repo.Migrations.FixLegacyTagsTest do + alias Pleroma.User + use Pleroma.DataCase + import Pleroma.Factory + import Pleroma.Tests.Helpers + + setup_all do: require_migration("20200802170532_fix_legacy_tags") + + test "change/0 converts legacy user tags into correct values", %{migration: migration} do + user = insert(:user, tags: ["force_nsfw", "force_unlisted", "verified"]) + user2 = insert(:user) + + assert :ok == migration.change() + + fixed_user = User.get_by_id(user.id) + fixed_user2 = User.get_by_id(user2.id) + + assert fixed_user.tags == ["mrf_tag:media-force-nsfw", "mrf_tag:force-unlisted", "verified"] + assert fixed_user2.tags == [] + + # user2 should not have been updated + assert fixed_user2.updated_at == fixed_user2.inserted_at + end +end From dc88b6f0919cf5686af7d5b935e8ee462491704b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 2 Aug 2020 14:53:42 -0500 Subject: [PATCH 07/32] Add email blacklist, fixes #1404 --- config/config.exs | 3 ++- config/description.exs | 7 +++++++ docs/configuration/cheatsheet.md | 5 +++++ lib/pleroma/user.ex | 11 ++++++++++- test/user_test.exs | 23 +++++++++++++++++++++++ 5 files changed, 47 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index d31208c25..ba263bf95 100644 --- a/config/config.exs +++ b/config/config.exs @@ -509,7 +509,8 @@ config :pleroma, Pleroma.User, "user_exists", "users", "web" - ] + ], + email_blacklist: [] config :pleroma, Oban, repo: Pleroma.Repo, diff --git a/config/description.exs b/config/description.exs index 11fbe0d78..3fe22e969 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3021,6 +3021,7 @@ config :pleroma, :config_description, [ %{ key: :restricted_nicknames, type: {:list, :string}, + description: "List of nicknames users may not register with.", suggestions: [ ".well-known", "~", @@ -3053,6 +3054,12 @@ config :pleroma, :config_description, [ "users", "web" ] + }, + %{ + key: :email_blacklist, + type: {:list, :string}, + description: "List of email domains users may not register with.", + suggestions: ["mailinator.com", "maildrop.cc"] } ] }, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 9c768abef..1a86179f3 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -202,6 +202,11 @@ config :pleroma, :mrf_user_allowlist, %{ * `sign_object_fetches`: Sign object fetches with HTTP signatures * `authorized_fetch_mode`: Require HTTP signatures for AP fetches +## Pleroma.User + +* `restricted_nicknames`: List of nicknames users may not register with. +* `email_blacklist`: List of email domains users may not register with. + ## Pleroma.ScheduledActivity * `daily_user_limit`: the number of scheduled activities a user is allowed to create in a single day (Default: `25`) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index dcf6ebee2..d0cc098fe 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -676,10 +676,19 @@ defmodule Pleroma.User do |> validate_required([:name, :nickname, :password, :password_confirmation]) |> validate_confirmation(:password) |> unique_constraint(:email) + |> validate_format(:email, @email_regex) + |> validate_change(:email, fn :email, email -> + valid? = + Config.get([User, :email_blacklist]) + |> Enum.all?(fn blacklisted_domain -> + !String.ends_with?(email, ["@" <> blacklisted_domain, "." <> blacklisted_domain]) + end) + + if valid?, do: [], else: [email: "Email domain is blacklisted"] + end) |> unique_constraint(:nickname) |> validate_exclusion(:nickname, Config.get([User, :restricted_nicknames])) |> validate_format(:nickname, local_nickname_regex()) - |> validate_format(:email, @email_regex) |> validate_length(:bio, max: bio_limit) |> validate_length(:name, min: 1, max: name_limit) |> validate_length(:registration_reason, max: reason_limit) diff --git a/test/user_test.exs b/test/user_test.exs index 904cea536..7c45e69e7 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -490,6 +490,29 @@ defmodule Pleroma.UserTest do refute changeset.valid? end + test "it blocks blacklisted email domains" do + clear_config([User, :email_blacklist], ["trolling.world"]) + + # Block with match + params = Map.put(@full_user_data, :email, "troll@trolling.world") + changeset = User.register_changeset(%User{}, params) + refute changeset.valid? + + # Block with subdomain match + params = Map.put(@full_user_data, :email, "troll@gnomes.trolling.world") + changeset = User.register_changeset(%User{}, params) + refute changeset.valid? + + # Pass with different domains that are similar + params = Map.put(@full_user_data, :email, "troll@gnomestrolling.world") + changeset = User.register_changeset(%User{}, params) + assert changeset.valid? + + params = Map.put(@full_user_data, :email, "troll@trolling.world.us") + changeset = User.register_changeset(%User{}, params) + assert changeset.valid? + end + test "it sets the password_hash and ap_id" do changeset = User.register_changeset(%User{}, @full_user_data) From de3bdc63adac0141500bdc2692124cd104330bda Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 3 Aug 2020 15:00:14 +0200 Subject: [PATCH 08/32] AccountControllerTest: Add test for message returned. --- .../controllers/account_controller_test.exs | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/test/web/mastodon_api/controllers/account_controller_test.exs b/test/web/mastodon_api/controllers/account_controller_test.exs index d390c3ce1..2cb388655 100644 --- a/test/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/web/mastodon_api/controllers/account_controller_test.exs @@ -940,17 +940,32 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do assert refresh assert scope == "read write follow" + clear_config([User, :email_blacklist], ["example.org"]) + + params = %{ + username: "lain", + email: "lain@example.org", + password: "PlzDontHackLain", + bio: "Test Bio", + agreement: true + } + conn = build_conn() |> put_req_header("content-type", "multipart/form-data") |> put_req_header("authorization", "Bearer " <> token) - |> post("/api/v1/accounts", %{ - username: "lain", - email: "lain@example.org", - password: "PlzDontHackLain", - bio: "Test Bio", - agreement: true - }) + |> post("/api/v1/accounts", params) + + assert %{"error" => "{\"email\":[\"Email domain is blacklisted\"]}"} = + json_response_and_validate_schema(conn, 400) + + Pleroma.Config.put([User, :email_blacklist], []) + + conn = + build_conn() + |> put_req_header("content-type", "multipart/form-data") + |> put_req_header("authorization", "Bearer " <> token) + |> post("/api/v1/accounts", params) %{ "access_token" => token, From 016d8d6c560cb81dfe67cc660e12d2e70d0bc6af Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 3 Aug 2020 12:37:31 -0500 Subject: [PATCH 09/32] Consolidate construction of Rich Media Parser HTTP requests --- lib/pleroma/web/rich_media/helpers.ex | 21 +++++++++++++++++++ lib/pleroma/web/rich_media/parser.ex | 20 +----------------- .../web/rich_media/parsers/oembed_parser.ex | 2 +- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 5c7daf1a5..6210f2c5a 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -9,6 +9,11 @@ defmodule Pleroma.Web.RichMedia.Helpers do alias Pleroma.Object alias Pleroma.Web.RichMedia.Parser + @rich_media_options [ + pool: :media, + max_body: 2_000_000 + ] + @spec validate_page_url(URI.t() | binary()) :: :ok | :error defp validate_page_url(page_url) when is_binary(page_url) do validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld]) @@ -77,4 +82,20 @@ defmodule Pleroma.Web.RichMedia.Helpers do fetch_data_for_activity(activity) :ok end + + def rich_media_get(url) do + headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}] + + options = + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do + Keyword.merge(@rich_media_options, + recv_timeout: 2_000, + with_body: true + ) + else + @rich_media_options + end + + Pleroma.HTTP.get(url, headers, options) + end end diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index c8a767935..ca592833f 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -3,11 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RichMedia.Parser do - @options [ - pool: :media, - max_body: 2_000_000 - ] - defp parsers do Pleroma.Config.get([:rich_media, :parsers]) end @@ -75,21 +70,8 @@ defmodule Pleroma.Web.RichMedia.Parser do end defp parse_url(url) do - opts = - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do - Keyword.merge(@options, - recv_timeout: 2_000, - with_body: true - ) - else - @options - end - try do - rich_media_agent = Pleroma.Application.user_agent() <> "; Bot" - - {:ok, %Tesla.Env{body: html}} = - Pleroma.HTTP.get(url, [{"user-agent", rich_media_agent}], adapter: opts) + {:ok, %Tesla.Env{body: html}} = Pleroma.Web.RichMedia.Helpers.rich_media_get(url) html |> parse_html() diff --git a/lib/pleroma/web/rich_media/parsers/oembed_parser.ex b/lib/pleroma/web/rich_media/parsers/oembed_parser.ex index 6bdeac89c..1fe6729c3 100644 --- a/lib/pleroma/web/rich_media/parsers/oembed_parser.ex +++ b/lib/pleroma/web/rich_media/parsers/oembed_parser.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Web.RichMedia.Parsers.OEmbed do end defp get_oembed_data(url) do - with {:ok, %Tesla.Env{body: json}} <- Pleroma.HTTP.get(url, [], adapter: [pool: :media]) do + with {:ok, %Tesla.Env{body: json}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url) do Jason.decode(json) end end From 058daf498f10e58221bd29a42799f52e56a800a9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 3 Aug 2020 19:57:53 -0500 Subject: [PATCH 10/32] Email blacklist: Update response phrasing --- lib/pleroma/user.ex | 2 +- test/web/mastodon_api/controllers/account_controller_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index d0cc098fe..16679ac42 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -684,7 +684,7 @@ defmodule Pleroma.User do !String.ends_with?(email, ["@" <> blacklisted_domain, "." <> blacklisted_domain]) end) - if valid?, do: [], else: [email: "Email domain is blacklisted"] + if valid?, do: [], else: [credentials: "Invalid credentials"] end) |> unique_constraint(:nickname) |> validate_exclusion(:nickname, Config.get([User, :restricted_nicknames])) diff --git a/test/web/mastodon_api/controllers/account_controller_test.exs b/test/web/mastodon_api/controllers/account_controller_test.exs index 2cb388655..86e3ac3e7 100644 --- a/test/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/web/mastodon_api/controllers/account_controller_test.exs @@ -956,7 +956,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do |> put_req_header("authorization", "Bearer " <> token) |> post("/api/v1/accounts", params) - assert %{"error" => "{\"email\":[\"Email domain is blacklisted\"]}"} = + assert %{"error" => "{\"credentials\":[\"Invalid credentials\"]}"} = json_response_and_validate_schema(conn, 400) Pleroma.Config.put([User, :email_blacklist], []) From 4f57e85ab9c80fb7cb51428cef978793ba22971c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 3 Aug 2020 22:20:49 -0500 Subject: [PATCH 11/32] Email blacklist: Update phrasing again --- lib/pleroma/user.ex | 2 +- test/web/mastodon_api/controllers/account_controller_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 16679ac42..9e03373de 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -684,7 +684,7 @@ defmodule Pleroma.User do !String.ends_with?(email, ["@" <> blacklisted_domain, "." <> blacklisted_domain]) end) - if valid?, do: [], else: [credentials: "Invalid credentials"] + if valid?, do: [], else: [email: "Invalid email"] end) |> unique_constraint(:nickname) |> validate_exclusion(:nickname, Config.get([User, :restricted_nicknames])) diff --git a/test/web/mastodon_api/controllers/account_controller_test.exs b/test/web/mastodon_api/controllers/account_controller_test.exs index 86e3ac3e7..17a1e7d66 100644 --- a/test/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/web/mastodon_api/controllers/account_controller_test.exs @@ -956,7 +956,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do |> put_req_header("authorization", "Bearer " <> token) |> post("/api/v1/accounts", params) - assert %{"error" => "{\"credentials\":[\"Invalid credentials\"]}"} = + assert %{"error" => "{\"email\":[\"Invalid email\"]}"} = json_response_and_validate_schema(conn, 400) Pleroma.Config.put([User, :email_blacklist], []) From 2f4289d455fbd2d949ac1e10d5ea2b9c78f15e82 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 4 Aug 2020 12:49:56 +0200 Subject: [PATCH 12/32] Changelog: Add info about email blacklist --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 129c269aa..6ae5fb928 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added +- Configuration: Added a blacklist for email servers. - Chats: Added `accepts_chat_messages` field to user, exposed in APIs and federation. - Chats: Added support for federated chats. For details, see the docs. - ActivityPub: Added support for existing AP ids for instances migrated from Mastodon. From 56e9bf33932bacfdffd700b97e3117fc593cac11 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Tue, 4 Aug 2020 14:35:47 +0300 Subject: [PATCH 13/32] Unify Config.get behaviour for atom/list key param --- lib/pleroma/config.ex | 34 +++++++++++++++++++++++++++------- test/config_test.exs | 28 ++++++++++++++++++++++++++++ test/support/helpers.ex | 14 ++++++++++++-- 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/config.ex b/lib/pleroma/config.ex index cc80deff5..88d1972ba 100644 --- a/lib/pleroma/config.ex +++ b/lib/pleroma/config.ex @@ -11,13 +11,33 @@ defmodule Pleroma.Config do def get([key], default), do: get(key, default) - def get([parent_key | keys], default) do - case :pleroma - |> Application.get_env(parent_key) - |> get_in(keys) do - nil -> default - any -> any - end + def get([root_key | keys], default) do + # This is to mimic Application.get_env/3 behaviour that returns `nil` if the + # actual value is `nil`. + Enum.reduce_while(keys, Application.get_env(:pleroma, root_key), fn key, config -> + case key do + [last_key] when is_map(config) -> + {:halt, Map.get(config, last_key, default)} + + [last_key] when is_list(config) -> + {:halt, Keyword.get(config, last_key, default)} + + _ -> + case config do + %{^key => value} -> + {:cont, value} + + [_ | _] -> + case :lists.keyfind(key, 1, config) do + {_, value} -> {:cont, value} + _ -> {:halt, default} + end + + _ -> + {:halt, default} + end + end + end) end def get(key, default) do diff --git a/test/config_test.exs b/test/config_test.exs index a46ab4302..3f3da06d0 100644 --- a/test/config_test.exs +++ b/test/config_test.exs @@ -28,6 +28,34 @@ defmodule Pleroma.ConfigTest do assert Pleroma.Config.get([:azerty, :uiop], true) == true end + describe "nil values" do + setup do + Pleroma.Config.put(:lorem, nil) + Pleroma.Config.put(:ipsum, %{dolor: [sit: nil]}) + Pleroma.Config.put(:dolor, sit: %{amet: nil}) + + on_exit(fn -> Enum.each(~w(lorem ipsum dolor)a, &Pleroma.Config.delete/1) end) + end + + test "get/1 with an atom for nil value" do + assert Pleroma.Config.get(:lorem) == nil + end + + test "get/2 with an atom for nil value" do + assert Pleroma.Config.get(:lorem, true) == nil + end + + test "get/1 with a list of keys for nil value" do + assert Pleroma.Config.get([:ipsum, :dolor, :sit]) == nil + assert Pleroma.Config.get([:dolor, :sit, :amet]) == nil + end + + test "get/2 with a list of keys for nil value" do + assert Pleroma.Config.get([:ipsum, :dolor, :sit], true) == nil + assert Pleroma.Config.get([:dolor, :sit, :amet], true) == nil + end + end + test "get/1 when value is false" do Pleroma.Config.put([:instance, :false_test], false) Pleroma.Config.put([:instance, :nested], []) diff --git a/test/support/helpers.ex b/test/support/helpers.ex index 5cbf2e291..7d729541d 100644 --- a/test/support/helpers.ex +++ b/test/support/helpers.ex @@ -17,9 +17,19 @@ defmodule Pleroma.Tests.Helpers do defmacro clear_config(config_path, do: yield) do quote do - initial_setting = Config.get(unquote(config_path)) + initial_setting = Config.get(unquote(config_path), :__clear_config_absent__) unquote(yield) - on_exit(fn -> Config.put(unquote(config_path), initial_setting) end) + + on_exit(fn -> + case initial_setting do + :__clear_config_absent__ -> + Config.delete(unquote(config_path)) + + _ -> + Config.put(unquote(config_path), initial_setting) + end + end) + :ok end end From 953f71bcfa25569d8b92d4047f4bdbee97e0077c Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 4 Aug 2020 13:38:30 +0200 Subject: [PATCH 14/32] App Test: Make more resilient --- test/tasks/app_test.exs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/tasks/app_test.exs b/test/tasks/app_test.exs index b8f03566d..71a84ac8e 100644 --- a/test/tasks/app_test.exs +++ b/test/tasks/app_test.exs @@ -50,13 +50,13 @@ defmodule Mix.Tasks.Pleroma.AppTest do defp assert_app(name, redirect, scopes) do app = Repo.get_by(Pleroma.Web.OAuth.App, client_name: name) - assert_received {:mix_shell, :info, [message]} + assert_receive {:mix_shell, :info, [message]} assert message == "#{name} successfully created:" - assert_received {:mix_shell, :info, [message]} + assert_receive {:mix_shell, :info, [message]} assert message == "App client_id: #{app.client_id}" - assert_received {:mix_shell, :info, [message]} + assert_receive {:mix_shell, :info, [message]} assert message == "App client_secret: #{app.client_secret}" assert app.scopes == scopes From 988ca4ab6a0d299308d96e84aa45ef63341128bf Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 4 Aug 2020 14:07:10 +0200 Subject: [PATCH 15/32] Test Config: Don't have any MRFs by default --- config/test.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/test.exs b/config/test.exs index db0655e73..413c7f0b9 100644 --- a/config/test.exs +++ b/config/test.exs @@ -120,6 +120,8 @@ config :pleroma, Pleroma.Uploaders.S3, config :tzdata, :autoupdate, :disabled +config :pleroma, :mrf, policies: [] + if File.exists?("./config/test.secret.exs") do import_config "test.secret.exs" else From e92c040ad3d0cc568ea0dc4b79f207a392c7c90f Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 4 Aug 2020 14:08:12 +0200 Subject: [PATCH 16/32] CommonAPITest: Add test that deactivated users can't post. --- test/web/common_api/common_api_test.exs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 313dda21b..4ba6232dc 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -458,6 +458,11 @@ defmodule Pleroma.Web.CommonAPITest do end describe "posting" do + test "deactivated users can't post" do + user = insert(:user, deactivated: true) + assert {:error, _} = CommonAPI.post(user, %{status: "ye"}) + end + test "it supports explicit addressing" do user = insert(:user) user_two = insert(:user) From 0cfadcf2caf84e2db944036576bad888a9707ff1 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 4 Aug 2020 14:15:32 +0200 Subject: [PATCH 17/32] TransmogrifierTest: Add test for deactivated users --- test/web/activity_pub/transmogrifier_test.exs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 7d33feaf2..828964a36 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -163,6 +163,14 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do end) =~ "[warn] Couldn't fetch \"https://404.site/whatever\", error: nil" end + test "it does not work for deactivated users" do + data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() + + insert(:user, ap_id: data["actor"], deactivated: true) + + assert {:error, _} = Transmogrifier.handle_incoming(data) + end + test "it works for incoming notices" do data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() From 1a00713744803824b16efd575c9c6880b1d1a57e Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 4 Aug 2020 14:17:03 +0200 Subject: [PATCH 18/32] CommonValidations: Treat deactivated users as not present. --- .../object_validators/common_validations.ex | 13 +++++++++---- .../transmogrifier/chat_message_test.exs | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex index aeef31945..bd46f8034 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex @@ -34,10 +34,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do cng |> validate_change(field_name, fn field_name, actor -> - if User.get_cached_by_ap_id(actor) do - [] - else - [{field_name, "can't find user"}] + case User.get_cached_by_ap_id(actor) do + %User{deactivated: true} -> + [{field_name, "user is deactivated"}] + + %User{} -> + [] + + _ -> + [{field_name, "can't find user"}] end end) end diff --git a/test/web/activity_pub/transmogrifier/chat_message_test.exs b/test/web/activity_pub/transmogrifier/chat_message_test.exs index d6736dc3e..31274c067 100644 --- a/test/web/activity_pub/transmogrifier/chat_message_test.exs +++ b/test/web/activity_pub/transmogrifier/chat_message_test.exs @@ -124,6 +124,24 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.ChatMessageTest do {:ok, %Activity{} = _activity} = Transmogrifier.handle_incoming(data) end + test "it doesn't work for deactivated users" do + data = + File.read!("test/fixtures/create-chat-message.json") + |> Poison.decode!() + + _author = + insert(:user, + ap_id: data["actor"], + local: false, + last_refreshed_at: DateTime.utc_now(), + deactivated: true + ) + + _recipient = insert(:user, ap_id: List.first(data["to"]), local: true) + + assert {:error, _} = Transmogrifier.handle_incoming(data) + end + test "it inserts it and creates a chat" do data = File.read!("test/fixtures/create-chat-message.json") From 36aa34a1a8c489f74a9821095d823f8060afac5f Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 4 Aug 2020 15:08:51 +0200 Subject: [PATCH 19/32] MastodonAPITest: Do the needful --- test/web/mastodon_api/mastodon_api_test.exs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/web/mastodon_api/mastodon_api_test.exs b/test/web/mastodon_api/mastodon_api_test.exs index c08be37d4..0c5a38bf6 100644 --- a/test/web/mastodon_api/mastodon_api_test.exs +++ b/test/web/mastodon_api/mastodon_api_test.exs @@ -17,8 +17,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPITest do test "returns error when followed user is deactivated" do follower = insert(:user) user = insert(:user, local: true, deactivated: true) - {:error, error} = MastodonAPI.follow(follower, user) - assert error == :rejected + assert {:error, _error} = MastodonAPI.follow(follower, user) end test "following for user" do From 697e3db01c0a1ee1e18fe25946a4ef56527828e7 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 4 Aug 2020 08:55:40 -0500 Subject: [PATCH 20/32] Add analyze mix alias to run the same credo checks we use in CI --- mix.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 0e723c15f..63142dee7 100644 --- a/mix.exs +++ b/mix.exs @@ -214,7 +214,8 @@ defmodule Pleroma.Mixfile do "ecto.setup": ["ecto.create", "ecto.migrate", "run priv/repo/seeds.exs"], "ecto.reset": ["ecto.drop", "ecto.setup"], test: ["ecto.create --quiet", "ecto.migrate", "test"], - docs: ["pleroma.docs", "docs"] + docs: ["pleroma.docs", "docs"], + analyze: ["credo --strict --only=warnings,todo,fixme,consistency,readability"] ] end From 91fbb5b21f9d8f098c9796eb4dd917bcd1e92404 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 4 Aug 2020 18:26:37 +0400 Subject: [PATCH 21/32] Fix ActivityExpirationPolicy --- .../web/activity_pub/mrf/activity_expiration_policy.ex | 4 ++-- .../activity_pub/mrf/activity_expiration_policy_test.exs | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex index 8e47f1e02..7b4c78e0f 100644 --- a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex @@ -21,8 +21,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy do @impl true def describe, do: {:ok, %{}} - defp local?(%{"id" => id}) do - String.starts_with?(id, Pleroma.Web.Endpoint.url()) + defp local?(%{"actor" => actor}) do + String.starts_with?(actor, Pleroma.Web.Endpoint.url()) end defp note?(activity) do diff --git a/test/web/activity_pub/mrf/activity_expiration_policy_test.exs b/test/web/activity_pub/mrf/activity_expiration_policy_test.exs index 8babf49e7..f25cf8b12 100644 --- a/test/web/activity_pub/mrf/activity_expiration_policy_test.exs +++ b/test/web/activity_pub/mrf/activity_expiration_policy_test.exs @@ -7,11 +7,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicyTest do alias Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy @id Pleroma.Web.Endpoint.url() <> "/activities/cofe" + @local_actor Pleroma.Web.Endpoint.url() <> "/users/cofe" test "adds `expires_at` property" do assert {:ok, %{"type" => "Create", "expires_at" => expires_at}} = ActivityExpirationPolicy.filter(%{ "id" => @id, + "actor" => @local_actor, "type" => "Create", "object" => %{"type" => "Note"} }) @@ -25,6 +27,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicyTest do assert {:ok, %{"type" => "Create", "expires_at" => ^expires_at}} = ActivityExpirationPolicy.filter(%{ "id" => @id, + "actor" => @local_actor, "type" => "Create", "expires_at" => expires_at, "object" => %{"type" => "Note"} @@ -37,6 +40,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicyTest do assert {:ok, %{"type" => "Create", "expires_at" => expires_at}} = ActivityExpirationPolicy.filter(%{ "id" => @id, + "actor" => @local_actor, "type" => "Create", "expires_at" => too_distant_future, "object" => %{"type" => "Note"} @@ -49,6 +53,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicyTest do assert {:ok, activity} = ActivityExpirationPolicy.filter(%{ "id" => "https://example.com/123", + "actor" => "https://example.com/users/cofe", "type" => "Create", "object" => %{"type" => "Note"} }) @@ -60,6 +65,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicyTest do assert {:ok, activity} = ActivityExpirationPolicy.filter(%{ "id" => "https://example.com/123", + "actor" => "https://example.com/users/cofe", "type" => "Follow" }) @@ -68,6 +74,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicyTest do assert {:ok, activity} = ActivityExpirationPolicy.filter(%{ "id" => "https://example.com/123", + "actor" => "https://example.com/users/cofe", "type" => "Create", "object" => %{"type" => "Cofe"} }) From 184742af5eed2c48ba8518f1e114cbe0655ad467 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 3 Aug 2020 22:32:51 -0500 Subject: [PATCH 22/32] Unique apps.client_id for new installations, fixes #2022 --- ...200804183107_add_unique_index_to_app_client_id.exs | 7 +++++++ test/web/oauth/app_test.exs | 11 +++++++++++ 2 files changed, 18 insertions(+) create mode 100644 priv/repo/migrations/20200804183107_add_unique_index_to_app_client_id.exs diff --git a/priv/repo/migrations/20200804183107_add_unique_index_to_app_client_id.exs b/priv/repo/migrations/20200804183107_add_unique_index_to_app_client_id.exs new file mode 100644 index 000000000..83de18096 --- /dev/null +++ b/priv/repo/migrations/20200804183107_add_unique_index_to_app_client_id.exs @@ -0,0 +1,7 @@ +defmodule Pleroma.Repo.Migrations.AddUniqueIndexToAppClientId do + use Ecto.Migration + + def change do + create(unique_index(:apps, [:client_id])) + end +end diff --git a/test/web/oauth/app_test.exs b/test/web/oauth/app_test.exs index 899af648e..993a490e0 100644 --- a/test/web/oauth/app_test.exs +++ b/test/web/oauth/app_test.exs @@ -29,5 +29,16 @@ defmodule Pleroma.Web.OAuth.AppTest do assert exist_app.id == app.id assert exist_app.scopes == ["read", "write", "follow", "push"] end + + test "has unique client_id" do + insert(:oauth_app, client_name: "", redirect_uris: "", client_id: "boop") + + error = + catch_error(insert(:oauth_app, client_name: "", redirect_uris: "", client_id: "boop")) + + assert %Ecto.ConstraintError{} = error + assert error.constraint == "apps_client_id_index" + assert error.type == :unique + end end end From 079e410d6efcb39e72a238c13e52bd1898b442a2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 4 Aug 2020 13:12:23 -0500 Subject: [PATCH 23/32] Add a migration to clean up activity_expirations table --- ...0804180322_remove_nonlocal_expirations.exs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 priv/repo/migrations/20200804180322_remove_nonlocal_expirations.exs diff --git a/priv/repo/migrations/20200804180322_remove_nonlocal_expirations.exs b/priv/repo/migrations/20200804180322_remove_nonlocal_expirations.exs new file mode 100644 index 000000000..389935f0d --- /dev/null +++ b/priv/repo/migrations/20200804180322_remove_nonlocal_expirations.exs @@ -0,0 +1,19 @@ +defmodule Pleroma.Repo.Migrations.RemoveNonlocalExpirations do + use Ecto.Migration + + def up do + statement = """ + DELETE FROM + activity_expirations A USING activities B + WHERE + A.activity_id = B.id + AND B.local = false; + """ + + execute(statement) + end + + def down do + :ok + end +end From 577b11167cb55203d30c43773f40108a87b2be6d Mon Sep 17 00:00:00 2001 From: Karol Kosek Date: Wed, 5 Aug 2020 00:01:30 +0200 Subject: [PATCH 24/32] templates/layout/app.html.eex: fix link color --- lib/pleroma/web/templates/layout/app.html.eex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/templates/layout/app.html.eex b/lib/pleroma/web/templates/layout/app.html.eex index 5836ec1e0..51603fe0c 100644 --- a/lib/pleroma/web/templates/layout/app.html.eex +++ b/lib/pleroma/web/templates/layout/app.html.eex @@ -37,7 +37,7 @@ } a { - color: color: #d8a070; + color: #d8a070; text-decoration: none; } From f341a8e142ad9d4c92afc4a97ef387df068e38e0 Mon Sep 17 00:00:00 2001 From: MK Fain Date: Wed, 5 Aug 2020 02:01:27 +0000 Subject: [PATCH 25/32] Update filter_view.ex to return whole_word actual value --- lib/pleroma/web/mastodon_api/views/filter_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/filter_view.ex b/lib/pleroma/web/mastodon_api/views/filter_view.ex index aeff646f5..c37f624e0 100644 --- a/lib/pleroma/web/mastodon_api/views/filter_view.ex +++ b/lib/pleroma/web/mastodon_api/views/filter_view.ex @@ -25,7 +25,7 @@ defmodule Pleroma.Web.MastodonAPI.FilterView do context: filter.context, expires_at: expires_at, irreversible: filter.hide, - whole_word: false + whole_word: filter.whole_word } end end From 6f60ac9f41d9511afa71986f000a2fc6c637b0c5 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Wed, 5 Aug 2020 13:00:49 +0300 Subject: [PATCH 26/32] Refactor config --- lib/pleroma/config.ex | 61 ++++++++++++-------------- test/application_requirements_test.exs | 5 ++- test/config_test.exs | 16 +++++++ 3 files changed, 49 insertions(+), 33 deletions(-) diff --git a/lib/pleroma/config.ex b/lib/pleroma/config.ex index 88d1972ba..98099ca58 100644 --- a/lib/pleroma/config.ex +++ b/lib/pleroma/config.ex @@ -11,33 +11,11 @@ defmodule Pleroma.Config do def get([key], default), do: get(key, default) - def get([root_key | keys], default) do - # This is to mimic Application.get_env/3 behaviour that returns `nil` if the - # actual value is `nil`. - Enum.reduce_while(keys, Application.get_env(:pleroma, root_key), fn key, config -> - case key do - [last_key] when is_map(config) -> - {:halt, Map.get(config, last_key, default)} - - [last_key] when is_list(config) -> - {:halt, Keyword.get(config, last_key, default)} - - _ -> - case config do - %{^key => value} -> - {:cont, value} - - [_ | _] -> - case :lists.keyfind(key, 1, config) do - {_, value} -> {:cont, value} - _ -> {:halt, default} - end - - _ -> - {:halt, default} - end - end - end) + def get([_ | _] = path, default) do + case fetch(path) do + {:ok, value} -> value + :error -> default + end end def get(key, default) do @@ -54,6 +32,22 @@ defmodule Pleroma.Config do end end + def fetch([root_key | keys]) do + Enum.reduce_while(keys, Application.fetch_env(:pleroma, root_key), fn + key, {:ok, config} when is_map(config) or is_list(config) -> + case Access.fetch(config, key) do + :error -> + {:halt, :error} + + value -> + {:cont, value} + end + + _key, _config -> + {:halt, :error} + end) + end + def put([key], value), do: put(key, value) def put([parent_key | keys], value) do @@ -70,12 +64,15 @@ defmodule Pleroma.Config do def delete([key]), do: delete(key) - def delete([parent_key | keys]) do - {_, parent} = - Application.get_env(:pleroma, parent_key) - |> get_and_update_in(keys, fn _ -> :pop end) + def delete([parent_key | keys] = path) do + with {:ok, _} <- fetch(path) do + {_, parent} = + parent_key + |> get() + |> get_and_update_in(keys, fn _ -> :pop end) - Application.put_env(:pleroma, parent_key, parent) + Application.put_env(:pleroma, parent_key, parent) + end end def delete(key) do diff --git a/test/application_requirements_test.exs b/test/application_requirements_test.exs index 21d24ddd0..e96295955 100644 --- a/test/application_requirements_test.exs +++ b/test/application_requirements_test.exs @@ -127,7 +127,10 @@ defmodule Pleroma.ApplicationRequirementsTest do :ok end - setup do: clear_config([:i_am_aware_this_may_cause_data_loss, :disable_migration_check]) + setup do + Pleroma.Config.get(:i_am_aware_this_may_cause_data_loss, 42) |> IO.inspect() + clear_config([:i_am_aware_this_may_cause_data_loss, :disable_migration_check]) + end test "raises if it detects unapplied migrations" do assert_raise Pleroma.ApplicationRequirements.VerifyError, diff --git a/test/config_test.exs b/test/config_test.exs index 3f3da06d0..e2c18304e 100644 --- a/test/config_test.exs +++ b/test/config_test.exs @@ -117,5 +117,21 @@ defmodule Pleroma.ConfigTest do Pleroma.Config.put([:delete_me, :delete_me], hello: "world", world: "Hello") Pleroma.Config.delete([:delete_me, :delete_me, :world]) assert Pleroma.Config.get([:delete_me, :delete_me]) == [hello: "world"] + + assert Pleroma.Config.delete([:this_key_does_not_exist]) + assert Pleroma.Config.delete([:non, :existing, :key]) + end + + test "fetch/1" do + Pleroma.Config.put([:lorem], :ipsum) + Pleroma.Config.put([:ipsum], dolor: :sit) + + assert Pleroma.Config.fetch([:lorem]) == {:ok, :ipsum} + assert Pleroma.Config.fetch([:ipsum, :dolor]) == {:ok, :sit} + assert Pleroma.Config.fetch([:lorem, :ipsum]) == :error + assert Pleroma.Config.fetch([:loremipsum]) == :error + + Pleroma.Config.delete([:lorem]) + Pleroma.Config.delete([:ipsum]) end end From 97b57014496003cabb416766457552ef854fa658 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Wed, 5 Aug 2020 17:46:14 +0300 Subject: [PATCH 27/32] Update clear_config macro --- test/application_requirements_test.exs | 5 +---- test/support/helpers.ex | 8 ++++---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/test/application_requirements_test.exs b/test/application_requirements_test.exs index e96295955..21d24ddd0 100644 --- a/test/application_requirements_test.exs +++ b/test/application_requirements_test.exs @@ -127,10 +127,7 @@ defmodule Pleroma.ApplicationRequirementsTest do :ok end - setup do - Pleroma.Config.get(:i_am_aware_this_may_cause_data_loss, 42) |> IO.inspect() - clear_config([:i_am_aware_this_may_cause_data_loss, :disable_migration_check]) - end + setup do: clear_config([:i_am_aware_this_may_cause_data_loss, :disable_migration_check]) test "raises if it detects unapplied migrations" do assert_raise Pleroma.ApplicationRequirements.VerifyError, diff --git a/test/support/helpers.ex b/test/support/helpers.ex index 7d729541d..ecd4b1e18 100644 --- a/test/support/helpers.ex +++ b/test/support/helpers.ex @@ -17,16 +17,16 @@ defmodule Pleroma.Tests.Helpers do defmacro clear_config(config_path, do: yield) do quote do - initial_setting = Config.get(unquote(config_path), :__clear_config_absent__) + initial_setting = Config.fetch(unquote(config_path)) unquote(yield) on_exit(fn -> case initial_setting do - :__clear_config_absent__ -> + :error -> Config.delete(unquote(config_path)) - _ -> - Config.put(unquote(config_path), initial_setting) + {:ok, value} -> + Config.put(unquote(config_path), value) end end) From 8c57a299b463b7e5916addbbd3571b35e1742ebd Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Wed, 5 Aug 2020 18:23:12 +0300 Subject: [PATCH 28/32] Handle non-list keys in Config.fetch/1 --- lib/pleroma/config.ex | 2 ++ test/config_test.exs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/lib/pleroma/config.ex b/lib/pleroma/config.ex index 98099ca58..a8329cc1e 100644 --- a/lib/pleroma/config.ex +++ b/lib/pleroma/config.ex @@ -32,6 +32,8 @@ defmodule Pleroma.Config do end end + def fetch(key) when is_atom(key), do: fetch([key]) + def fetch([root_key | keys]) do Enum.reduce_while(keys, Application.fetch_env(:pleroma, root_key), fn key, {:ok, config} when is_map(config) or is_list(config) -> diff --git a/test/config_test.exs b/test/config_test.exs index e2c18304e..1556e4237 100644 --- a/test/config_test.exs +++ b/test/config_test.exs @@ -127,9 +127,11 @@ defmodule Pleroma.ConfigTest do Pleroma.Config.put([:ipsum], dolor: :sit) assert Pleroma.Config.fetch([:lorem]) == {:ok, :ipsum} + assert Pleroma.Config.fetch(:lorem) == {:ok, :ipsum} assert Pleroma.Config.fetch([:ipsum, :dolor]) == {:ok, :sit} assert Pleroma.Config.fetch([:lorem, :ipsum]) == :error assert Pleroma.Config.fetch([:loremipsum]) == :error + assert Pleroma.Config.fetch(:loremipsum) == :error Pleroma.Config.delete([:lorem]) Pleroma.Config.delete([:ipsum]) From 7569f225f1d43c6435eda6b62fd5eff3cd3408e0 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Wed, 5 Aug 2020 19:38:55 +0300 Subject: [PATCH 29/32] Move checks to application startup --- lib/pleroma/application.ex | 18 ++++++++++++++++++ lib/pleroma/upload/filter/exiftool.ex | 14 +++++++++----- lib/pleroma/upload/filter/mogrifun.ex | 8 +++++--- lib/pleroma/upload/filter/mogrify.ex | 12 ++++++------ 4 files changed, 38 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 0ffb55358..c0b5db9f1 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -47,6 +47,7 @@ defmodule Pleroma.Application do Pleroma.ApplicationRequirements.verify!() setup_instrumenters() load_custom_modules() + check_system_commands() Pleroma.Docs.JSON.compile() adapter = Application.get_env(:tesla, :adapter) @@ -249,4 +250,21 @@ defmodule Pleroma.Application do end defp http_children(_, _), do: [] + + defp check_system_commands do + filters = Config.get([Pleroma.Upload, :filters]) + + check_filter = fn filter, command_required -> + with true <- filter in filters, + false <- Pleroma.Utils.command_available?(command_required) do + Logger.error( + "#{filter} is specified in list of Pleroma.Upload filters, but the #{command_required} command is not found" + ) + end + end + + check_filter.(Pleroma.Upload.Filters.Exiftool, "exiftool") + check_filter.(Pleroma.Upload.Filters.Mogrify, "mogrify") + check_filter.(Pleroma.Upload.Filters.Mogrifun, "mogrify") + end end diff --git a/lib/pleroma/upload/filter/exiftool.ex b/lib/pleroma/upload/filter/exiftool.ex index e1b976c98..ea8798fe3 100644 --- a/lib/pleroma/upload/filter/exiftool.ex +++ b/lib/pleroma/upload/filter/exiftool.ex @@ -9,12 +9,16 @@ defmodule Pleroma.Upload.Filter.Exiftool do """ @behaviour Pleroma.Upload.Filter + @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()} def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do - if Pleroma.Utils.command_available?("exiftool") do - System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) - :ok - else - {:error, "exiftool command not found"} + try do + case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) do + {_response, 0} -> :ok + {error, 1} -> {:error, error} + end + rescue + _e in ErlangError -> + {:error, "exiftool command not found"} end end diff --git a/lib/pleroma/upload/filter/mogrifun.ex b/lib/pleroma/upload/filter/mogrifun.ex index 8f362333d..a8503ac24 100644 --- a/lib/pleroma/upload/filter/mogrifun.ex +++ b/lib/pleroma/upload/filter/mogrifun.ex @@ -34,12 +34,14 @@ defmodule Pleroma.Upload.Filter.Mogrifun do [{"fill", "yellow"}, {"tint", "40"}] ] + @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()} def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do - if Pleroma.Utils.command_available?("mogrify") do + try do Filter.Mogrify.do_filter(file, [Enum.random(@filters)]) :ok - else - {:error, "mogrify command not found"} + rescue + _e in ErlangError -> + {:error, "mogrify command not found"} end end diff --git a/lib/pleroma/upload/filter/mogrify.ex b/lib/pleroma/upload/filter/mogrify.ex index 4bd0c2eb4..7a45add5a 100644 --- a/lib/pleroma/upload/filter/mogrify.ex +++ b/lib/pleroma/upload/filter/mogrify.ex @@ -8,14 +8,14 @@ defmodule Pleroma.Upload.Filter.Mogrify do @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()} @type conversions :: conversion() | [conversion()] + @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()} def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do - if Pleroma.Utils.command_available?("mogrify") do - filters = Pleroma.Config.get!([__MODULE__, :args]) - - do_filter(file, filters) + try do + do_filter(file, Pleroma.Config.get!([__MODULE__, :args])) :ok - else - {:error, "mogrify command not found"} + rescue + _e in ErlangError -> + {:error, "mogrify command not found"} end end From d6ab9f2132cdcbed303c9ef0941bf7210e49c5d6 Mon Sep 17 00:00:00 2001 From: Mary Kate Date: Wed, 5 Aug 2020 15:36:25 -0500 Subject: [PATCH 30/32] update test for whole_word in filter --- .../controllers/filter_controller_test.exs | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/test/web/mastodon_api/controllers/filter_controller_test.exs b/test/web/mastodon_api/controllers/filter_controller_test.exs index f29547d13..6e94150b4 100644 --- a/test/web/mastodon_api/controllers/filter_controller_test.exs +++ b/test/web/mastodon_api/controllers/filter_controller_test.exs @@ -64,11 +64,13 @@ defmodule Pleroma.Web.MastodonAPI.FilterControllerTest do test "get a filter" do %{user: user, conn: conn} = oauth_access(["read:filters"]) + # check whole_word true query = %Pleroma.Filter{ user_id: user.id, filter_id: 2, phrase: "knight", - context: ["home"] + context: ["home"], + whole_word: false } {:ok, filter} = Pleroma.Filter.create(query) @@ -76,6 +78,25 @@ defmodule Pleroma.Web.MastodonAPI.FilterControllerTest do conn = get(conn, "/api/v1/filters/#{filter.filter_id}") assert response = json_response_and_validate_schema(conn, 200) + assert response["whole_word"] == false + + # check whole_word false + %{user: user, conn: conn} = oauth_access(["read:filters"]) + + query = %Pleroma.Filter{ + user_id: user.id, + filter_id: 3, + phrase: "knight", + context: ["home"], + whole_word: true + } + + {:ok, filter} = Pleroma.Filter.create(query) + + conn = get(conn, "/api/v1/filters/#{filter.filter_id}") + + assert response = json_response_and_validate_schema(conn, 200) + assert response["whole_word"] == true end test "update a filter" do @@ -86,7 +107,8 @@ defmodule Pleroma.Web.MastodonAPI.FilterControllerTest do filter_id: 2, phrase: "knight", context: ["home"], - hide: true + hide: true, + whole_word: true } {:ok, _filter} = Pleroma.Filter.create(query) @@ -108,6 +130,7 @@ defmodule Pleroma.Web.MastodonAPI.FilterControllerTest do assert response["phrase"] == new.phrase assert response["context"] == new.context assert response["irreversible"] == true + assert response["whole_word"] == true end test "delete a filter" do From f785dba09bc6c6624c17350356632d008f701183 Mon Sep 17 00:00:00 2001 From: Mary Kate Date: Wed, 5 Aug 2020 15:39:11 -0500 Subject: [PATCH 31/32] changelog for filter whole_word fix --- CHANGELOG.md | 1 + test/web/mastodon_api/controllers/filter_controller_test.exs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de017e30a..572f9e84b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,6 +102,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Fix CSP policy generation to include remote Captcha services - Fix edge case where MediaProxy truncates media, usually caused when Caddy is serving content for the other Federated instance. - Emoji Packs could not be listed when instance was set to `public: false` +- Fix whole_word always returning false on filter get requests ## [Unreleased (patch)] diff --git a/test/web/mastodon_api/controllers/filter_controller_test.exs b/test/web/mastodon_api/controllers/filter_controller_test.exs index 6e94150b4..0d426ec34 100644 --- a/test/web/mastodon_api/controllers/filter_controller_test.exs +++ b/test/web/mastodon_api/controllers/filter_controller_test.exs @@ -64,7 +64,7 @@ defmodule Pleroma.Web.MastodonAPI.FilterControllerTest do test "get a filter" do %{user: user, conn: conn} = oauth_access(["read:filters"]) - # check whole_word true + # check whole_word false query = %Pleroma.Filter{ user_id: user.id, filter_id: 2, @@ -80,7 +80,7 @@ defmodule Pleroma.Web.MastodonAPI.FilterControllerTest do assert response = json_response_and_validate_schema(conn, 200) assert response["whole_word"] == false - # check whole_word false + # check whole_word true %{user: user, conn: conn} = oauth_access(["read:filters"]) query = %Pleroma.Filter{ From 135ae4e35a3e6a084eb611ce3a21c7a6c6bba9fc Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 6 Aug 2020 16:00:00 +0300 Subject: [PATCH 32/32] [#2025] Defaulted OAuth login scopes choice to all scopes when user selects no scopes. --- lib/pleroma/web/oauth/oauth_controller.ex | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index f29b3cb57..dd00600ea 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -76,6 +76,13 @@ defmodule Pleroma.Web.OAuth.OAuthController do available_scopes = (app && app.scopes) || [] scopes = Scopes.fetch_scopes(params, available_scopes) + scopes = + if scopes == [] do + available_scopes + else + scopes + end + # Note: `params` might differ from `conn.params`; use `@params` not `@conn.params` in template render(conn, Authenticator.auth_template(), %{ response_type: params["response_type"],