From c7dc5ce85c5b143df45031d5c87eee0689da3713 Mon Sep 17 00:00:00 2001 From: silverpill Date: Thu, 29 Dec 2022 01:12:32 +0000 Subject: [PATCH 01/89] TagValidator: Allow unrecognized Tag types --- .../web/activity_pub/object_validators/tag_validator.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex index 9f15f1981..ad968994e 100644 --- a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex @@ -68,6 +68,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do |> validate_required([:type, :name, :icon]) end + # Fallback + def changeset(struct, data), do: cast(struct, data, [:type, :name]) + def icon_changeset(struct, data) do struct |> cast(data, [:type, :url]) From 45646ff52cd045485c6be92713edcd0b37225bc3 Mon Sep 17 00:00:00 2001 From: silverpill Date: Thu, 29 Dec 2022 23:57:19 +0000 Subject: [PATCH 02/89] TagValidator: Add test for Link tag --- test/fixtures/fep-e232.json | 26 +++++++++++++++++++ .../web/activity_pub/transmogrifier_test.exs | 16 ++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 test/fixtures/fep-e232.json diff --git a/test/fixtures/fep-e232.json b/test/fixtures/fep-e232.json new file mode 100644 index 000000000..98f339589 --- /dev/null +++ b/test/fixtures/fep-e232.json @@ -0,0 +1,26 @@ +{ + "@context": "https://www.w3.org/ns/activitystreams", + "type": "Create", + "actor": "https://example.org/users/alice", + "object": { + "id": "https://example.org/objects/10", + "type": "Note", + "attributedTo": "https://example.org/users/alice", + "content": "

test https://example.org/objects/9

", + "published": "2022-10-01T21:30:05.211215Z", + "tag": [ + { + "name": "https://example.org/objects/9", + "type": "Link", + "href": "https://example.org/objects/9", + "mediaType": "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"" + } + ], + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [ + "https://example.org/users/alice/followers" + ] + } +} diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index 6b4636d22..c0425f30c 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -123,6 +123,22 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert activity.data["context"] == object.data["context"] end + + # https://codeberg.org/fediverse/fep/src/branch/main/feps/fep-e232.md + test "it accepts FEP-e232 link tags" do + insert(:user, ap_id: "https://example.org/users/alice") + + message = File.read!("test/fixtures/fep-e232.json") |> Jason.decode!() + + assert {:ok, activity} = Transmogrifier.handle_incoming(message) + + object = Object.normalize(activity) + assert length(object.data["tag"]) == 1 + + tag = object.data["tag"] |> List.first() + assert tag["type"] == "Link" + assert tag["name"] == "https://example.org/objects/9" + end end describe "prepare outgoing" do From 5cfb0578a6845db377b5679ac05aa25ee5656211 Mon Sep 17 00:00:00 2001 From: silverpill Date: Fri, 30 Dec 2022 17:26:43 +0000 Subject: [PATCH 03/89] TagValidator: Drop unrecognized tags --- .../web/activity_pub/object_validators/tag_validator.ex | 7 +++++-- test/fixtures/fep-e232.json | 5 +++++ test/pleroma/web/activity_pub/transmogrifier_test.exs | 6 ++---- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex index ad968994e..cfd510c19 100644 --- a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex @@ -68,8 +68,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do |> validate_required([:type, :name, :icon]) end - # Fallback - def changeset(struct, data), do: cast(struct, data, [:type, :name]) + def changeset(struct, %{"type" => _} = data) do + struct + |> cast(data, []) + |> Map.put(:action, :ignore) + end def icon_changeset(struct, data) do struct diff --git a/test/fixtures/fep-e232.json b/test/fixtures/fep-e232.json index 98f339589..e9d12ae35 100644 --- a/test/fixtures/fep-e232.json +++ b/test/fixtures/fep-e232.json @@ -9,6 +9,11 @@ "content": "

test https://example.org/objects/9

", "published": "2022-10-01T21:30:05.211215Z", "tag": [ + { + "name": "@bob@example.net", + "type": "Mention", + "href": "https://example.net/users/bob" + }, { "name": "https://example.org/objects/9", "type": "Link", diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index c0425f30c..f76606479 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -124,8 +124,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert activity.data["context"] == object.data["context"] end - # https://codeberg.org/fediverse/fep/src/branch/main/feps/fep-e232.md - test "it accepts FEP-e232 link tags" do + test "it drops link tags" do insert(:user, ap_id: "https://example.org/users/alice") message = File.read!("test/fixtures/fep-e232.json") |> Jason.decode!() @@ -136,8 +135,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert length(object.data["tag"]) == 1 tag = object.data["tag"] |> List.first() - assert tag["type"] == "Link" - assert tag["name"] == "https://example.org/objects/9" + assert tag["type"] == "Mention" end end From d3b27d45a9115c04d4e469d1a9dc372ff8576d52 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 29 Mar 2023 23:18:44 +0300 Subject: [PATCH 04/89] List installed frontend refs in admin API --- docs/development/API/admin_api.md | 2 ++ .../web/admin_api/controllers/frontend_controller.ex | 12 +++++++++++- lib/pleroma/web/admin_api/views/frontend_view.ex | 3 ++- .../api_spec/operations/admin/frontend_operation.ex | 3 ++- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/development/API/admin_api.md b/docs/development/API/admin_api.md index f6e9f7d2a..7d31ee262 100644 --- a/docs/development/API/admin_api.md +++ b/docs/development/API/admin_api.md @@ -1585,6 +1585,7 @@ Returns the content of the document "build_url": "https://git.pleroma.social/pleroma/fedi-fe/-/jobs/artifacts/${ref}/download?job=build", "git": "https://git.pleroma.social/pleroma/fedi-fe", "installed": true, + "installed_refs": ["master"], "name": "fedi-fe", "ref": "master" }, @@ -1592,6 +1593,7 @@ Returns the content of the document "build_url": "https://git.pleroma.social/lambadalambda/kenoma/-/jobs/artifacts/${ref}/download?job=build", "git": "https://git.pleroma.social/lambadalambda/kenoma", "installed": false, + "installed_refs": [], "name": "kenoma", "ref": "master" } diff --git a/lib/pleroma/web/admin_api/controllers/frontend_controller.ex b/lib/pleroma/web/admin_api/controllers/frontend_controller.ex index b4dbb82fe..145c1fa4a 100644 --- a/lib/pleroma/web/admin_api/controllers/frontend_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/frontend_controller.ex @@ -22,7 +22,9 @@ defmodule Pleroma.Web.AdminAPI.FrontendController do [:frontends, :available] |> Config.get([]) |> Enum.map(fn {name, desc} -> - Map.put(desc, "installed", name in installed) + desc + |> Map.put("installed", name in installed) + |> Map.put("installed_refs", installed_refs(name)) end) render(conn, "index.json", frontends: frontends) @@ -43,4 +45,12 @@ defmodule Pleroma.Web.AdminAPI.FrontendController do [] end end + + def installed_refs(name) do + if name in installed() do + File.ls!(Path.join(Pleroma.Frontend.dir(), name)) + else + [] + end + end end diff --git a/lib/pleroma/web/admin_api/views/frontend_view.ex b/lib/pleroma/web/admin_api/views/frontend_view.ex index 0ca3d67cb..ae4016581 100644 --- a/lib/pleroma/web/admin_api/views/frontend_view.ex +++ b/lib/pleroma/web/admin_api/views/frontend_view.ex @@ -15,7 +15,8 @@ defmodule Pleroma.Web.AdminAPI.FrontendView do git: frontend["git"], build_url: frontend["build_url"], ref: frontend["ref"], - installed: frontend["installed"] + installed: frontend["installed"], + installed_refs: frontend["installed_refs"] } end end diff --git a/lib/pleroma/web/api_spec/operations/admin/frontend_operation.ex b/lib/pleroma/web/api_spec/operations/admin/frontend_operation.ex index 4bfe5ac5a..89b1c474c 100644 --- a/lib/pleroma/web/api_spec/operations/admin/frontend_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/frontend_operation.ex @@ -52,7 +52,8 @@ defmodule Pleroma.Web.ApiSpec.Admin.FrontendOperation do git: %Schema{type: :string, format: :uri, nullable: true}, build_url: %Schema{type: :string, format: :uri, nullable: true}, ref: %Schema{type: :string}, - installed: %Schema{type: :boolean} + installed: %Schema{type: :boolean}, + installed_refs: %Schema{type: :array, items: %Schema{type: :string}} } } } From 3037d2780c735846bc82994f620c970b104362a7 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 30 Mar 2023 11:16:40 +0300 Subject: [PATCH 05/89] Also list frontends that are not in the config file --- .../admin_api/controllers/frontend_controller.ex | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/admin_api/controllers/frontend_controller.ex b/lib/pleroma/web/admin_api/controllers/frontend_controller.ex index 145c1fa4a..9e2ed4aac 100644 --- a/lib/pleroma/web/admin_api/controllers/frontend_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/frontend_controller.ex @@ -18,15 +18,24 @@ defmodule Pleroma.Web.AdminAPI.FrontendController do def index(conn, _params) do installed = installed() + # FIrst get frontends from config, + # then add frontends that are installed but not in the config frontends = - [:frontends, :available] - |> Config.get([]) + Config.get([:frontends, :available], []) |> Enum.map(fn {name, desc} -> desc |> Map.put("installed", name in installed) |> Map.put("installed_refs", installed_refs(name)) end) + frontends = + frontends ++ + (installed + |> Enum.filter(fn n -> not Enum.any?(frontends, fn f -> f["name"] == n end) end) + |> Enum.map(fn name -> + %{"name" => name, "installed" => true, "installed_refs" => installed_refs(name)} + end)) + render(conn, "index.json", frontends: frontends) end From 22b72cd6b87b4a684bde9c34741e5bb1e1204ed0 Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Sun, 5 Mar 2023 06:37:59 +0000 Subject: [PATCH 06/89] Merge branch 'tusooa/oban-common-pipeline' into 'develop' Stop oban from retrying if validating errors occur when processing incoming data See merge request pleroma/pleroma!3844 --- lib/pleroma/workers/receiver_worker.ex | 3 ++ test/pleroma/web/federator_test.exs | 2 +- test/pleroma/workers/receiver_worker_test.exs | 29 ++++++++++++++++++- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index 4f513b907..cf1bb62b4 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -13,6 +13,9 @@ defmodule Pleroma.Workers.ReceiverWorker do {:ok, res} else {:error, :origin_containment_failed} -> {:cancel, :origin_containment_failed} + {:error, :already_present} -> {:cancel, :already_present} + {:error, {:validate_object, reason}} -> {:cancel, reason} + {:error, {:error, {:validate, reason}}} -> {:cancel, reason} {:error, {:reject, reason}} -> {:cancel, reason} e -> e end diff --git a/test/pleroma/web/federator_test.exs b/test/pleroma/web/federator_test.exs index 41d1c5d5e..1ffe6aae1 100644 --- a/test/pleroma/web/federator_test.exs +++ b/test/pleroma/web/federator_test.exs @@ -133,7 +133,7 @@ defmodule Pleroma.Web.FederatorTest do assert {:ok, _activity} = ObanHelpers.perform(job) assert {:ok, job} = Federator.incoming_ap_doc(params) - assert {:error, :already_present} = ObanHelpers.perform(job) + assert {:cancel, :already_present} = ObanHelpers.perform(job) end test "rejects incoming AP docs with incorrect origin" do diff --git a/test/pleroma/workers/receiver_worker_test.exs b/test/pleroma/workers/receiver_worker_test.exs index 283beee4d..acea0ae00 100644 --- a/test/pleroma/workers/receiver_worker_test.exs +++ b/test/pleroma/workers/receiver_worker_test.exs @@ -11,7 +11,7 @@ defmodule Pleroma.Workers.ReceiverWorkerTest do alias Pleroma.Workers.ReceiverWorker - test "it ignores MRF reject" do + test "it does not retry MRF reject" do params = insert(:note).data with_mock Pleroma.Web.ActivityPub.Transmogrifier, @@ -22,4 +22,31 @@ defmodule Pleroma.Workers.ReceiverWorkerTest do }) end end + + test "it does not retry ObjectValidator reject" do + params = + insert(:note_activity).data + |> Map.put("id", Pleroma.Web.ActivityPub.Utils.generate_activity_id()) + |> Map.put("object", %{ + "type" => "Note", + "id" => Pleroma.Web.ActivityPub.Utils.generate_object_id() + }) + + with_mock Pleroma.Web.ActivityPub.ObjectValidator, [:passthrough], + validate: fn _, _ -> {:error, %Ecto.Changeset{}} end do + assert {:cancel, {:error, %Ecto.Changeset{}}} = + ReceiverWorker.perform(%Oban.Job{ + args: %{"op" => "incoming_ap_doc", "params" => params} + }) + end + end + + test "it does not retry duplicates" do + params = insert(:note_activity).data + + assert {:cancel, :already_present} = + ReceiverWorker.perform(%Oban.Job{ + args: %{"op" => "incoming_ap_doc", "params" => params} + }) + end end From d640df39276e19c1adc31d46692e46b7cdcfb991 Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Mon, 6 Mar 2023 21:08:27 +0000 Subject: [PATCH 07/89] Merge branch 'fix/static-fe-feed-500' into 'develop' fix: remove static_fe pipeline for /users/:nickname/feed See merge request pleroma/pleroma!3852 --- lib/pleroma/web/router.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index ba1d64ab2..c1a690e28 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -835,8 +835,7 @@ defmodule Pleroma.Web.Router do end scope "/", Pleroma.Web do - # Note: html format is supported only if static FE is enabled - pipe_through([:accepts_html_xml, :static_fe]) + pipe_through([:accepts_html_xml]) get("/users/:nickname/feed", Feed.UserController, :feed, as: :user_feed) end From 937df7e46536fee9703285e8a136bba67f1f907b Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Mon, 6 Mar 2023 22:55:24 +0000 Subject: [PATCH 08/89] Merge branch 'fix/tag-feed-crashes' into 'develop' fix: atom/rss feed issues Closes #3045 See merge request pleroma/pleroma!3851 --- lib/pleroma/web/feed/feed_view.ex | 10 ++--- lib/pleroma/web/metadata/utils.ex | 5 ++- .../templates/feed/feed/_activity.atom.eex | 4 +- .../web/templates/feed/feed/_activity.rss.eex | 2 +- .../feed/feed/_tag_activity.atom.eex | 4 +- .../templates/feed/feed/_tag_activity.xml.eex | 2 +- .../pleroma/web/feed/user_controller_test.exs | 44 +++++++++++++++++-- test/pleroma/web/metadata/utils_test.exs | 2 +- 8 files changed, 56 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex index 449659f4b..034722eb2 100644 --- a/lib/pleroma/web/feed/feed_view.ex +++ b/lib/pleroma/web/feed/feed_view.ex @@ -6,7 +6,6 @@ defmodule Pleroma.Web.Feed.FeedView do use Phoenix.HTML use Pleroma.Web, :view - alias Pleroma.Formatter alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.Gettext @@ -72,7 +71,9 @@ defmodule Pleroma.Web.Feed.FeedView do def last_activity(activities), do: List.last(activities) - def activity_title(%{"content" => content, "summary" => summary} = data, opts \\ %{}) do + def activity_title(%{"content" => content} = data, opts \\ %{}) do + summary = Map.get(data, "summary", "") + title = cond do summary != "" -> summary @@ -81,9 +82,8 @@ defmodule Pleroma.Web.Feed.FeedView do end title - |> Pleroma.Web.Metadata.Utils.scrub_html() - |> Pleroma.Emoji.Formatter.demojify() - |> Formatter.truncate(opts[:max_length], opts[:omission]) + |> Pleroma.Web.Metadata.Utils.scrub_html_and_truncate(opts[:max_length], opts[:omission]) + |> HtmlEntities.encode() end def activity_description(data) do diff --git a/lib/pleroma/web/metadata/utils.ex b/lib/pleroma/web/metadata/utils.ex index 15414a988..80a8be9a2 100644 --- a/lib/pleroma/web/metadata/utils.ex +++ b/lib/pleroma/web/metadata/utils.ex @@ -30,12 +30,13 @@ defmodule Pleroma.Web.Metadata.Utils do |> scrub_html_and_truncate_object_field(object) end - def scrub_html_and_truncate(content, max_length \\ 200) when is_binary(content) do + def scrub_html_and_truncate(content, max_length \\ 200, omission \\ "...") + when is_binary(content) do content |> scrub_html |> Emoji.Formatter.demojify() |> HtmlEntities.decode() - |> Formatter.truncate(max_length) + |> Formatter.truncate(max_length, omission) end def scrub_html(content) when is_binary(content) do diff --git a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex index 260338772..b774f7984 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex @@ -4,8 +4,8 @@ <%= @data["id"] %> <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> <%= activity_description(@data) %> - <%= to_rfc3339(@activity.data["published"]) %> - <%= to_rfc3339(@activity.data["published"]) %> + <%= to_rfc3339(@data["published"]) %> + <%= to_rfc3339(@data["published"]) %> <%= activity_context(@activity) %> diff --git a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex index 5c8f35fe4..7de98f736 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex @@ -4,7 +4,7 @@ <%= @data["id"] %> <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> <%= activity_description(@data) %> - <%= to_rfc2822(@activity.data["published"]) %> + <%= to_rfc2822(@data["published"]) %> <%= activity_context(@activity) %> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex index 25980c1e4..03c222975 100644 --- a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex @@ -7,8 +7,8 @@ <%= @data["id"] %> <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> <%= activity_description(@data) %> - <%= to_rfc3339(@activity.data["published"]) %> - <%= to_rfc3339(@activity.data["published"]) %> + <%= to_rfc3339(@data["published"]) %> + <%= to_rfc3339(@data["published"]) %> <%= activity_context(@activity) %> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex index d582c83e8..1b8c34b87 100644 --- a/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex +++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex @@ -4,7 +4,7 @@ <%= activity_context(@activity) %> <%= activity_context(@activity) %> - <%= to_rfc2822(@activity.data["published"]) %> + <%= to_rfc2822(@data["published"]) %> <%= activity_description(@data) %> <%= for attachment <- @data["attachment"] || [] do %> diff --git a/test/pleroma/web/feed/user_controller_test.exs b/test/pleroma/web/feed/user_controller_test.exs index de32d3d4b..d3c4108de 100644 --- a/test/pleroma/web/feed/user_controller_test.exs +++ b/test/pleroma/web/feed/user_controller_test.exs @@ -57,9 +57,23 @@ defmodule Pleroma.Web.Feed.UserControllerTest do ) note_activity2 = insert(:note_activity, note: note2) + + note3 = + insert(:note, + user: user, + data: %{ + "content" => "This note tests whether HTML entities are truncated properly", + "summary" => "Won't, didn't fail", + "inReplyTo" => note_activity2.id + } + ) + + _note_activity3 = insert(:note_activity, note: note3) object = Object.normalize(note_activity, fetch: false) - [user: user, object: object, max_id: note_activity2.id] + encoded_title = FeedView.activity_title(note3.data) + + [user: user, object: object, max_id: note_activity2.id, encoded_title: encoded_title] end test "gets an atom feed", %{conn: conn, user: user, object: object, max_id: max_id} do @@ -74,7 +88,7 @@ defmodule Pleroma.Web.Feed.UserControllerTest do |> SweetXml.parse() |> SweetXml.xpath(~x"//entry/title/text()"l) - assert activity_titles == ['2hu', '2hu & as'] + assert activity_titles == ['Won\'t, didn\'...', '2hu', '2hu & as'] assert resp =~ FeedView.escape(object.data["content"]) assert resp =~ FeedView.escape(object.data["summary"]) assert resp =~ FeedView.escape(object.data["context"]) @@ -105,7 +119,7 @@ defmodule Pleroma.Web.Feed.UserControllerTest do |> SweetXml.parse() |> SweetXml.xpath(~x"//item/title/text()"l) - assert activity_titles == ['2hu', '2hu & as'] + assert activity_titles == ['Won\'t, didn\'...', '2hu', '2hu & as'] assert resp =~ FeedView.escape(object.data["content"]) assert resp =~ FeedView.escape(object.data["summary"]) assert resp =~ FeedView.escape(object.data["context"]) @@ -176,6 +190,30 @@ defmodule Pleroma.Web.Feed.UserControllerTest do |> get("/users/#{user.nickname}/feed.rss") |> response(200) end + + test "does not mangle HTML entities midway", %{ + conn: conn, + user: user, + object: object, + encoded_title: encoded_title + } do + resp = + conn + |> put_req_header("accept", "application/atom+xml") + |> get(user_feed_path(conn, :feed, user.nickname)) + |> response(200) + + activity_titles = + resp + |> SweetXml.parse() + |> SweetXml.xpath(~x"//entry/title/text()"l) + + assert activity_titles == ['Won\'t, didn\'...', '2hu', '2hu & as'] + assert resp =~ FeedView.escape(object.data["content"]) + assert resp =~ FeedView.escape(object.data["summary"]) + assert resp =~ FeedView.escape(object.data["context"]) + assert resp =~ encoded_title + end end # Note: see ActivityPubControllerTest for JSON format tests diff --git a/test/pleroma/web/metadata/utils_test.exs b/test/pleroma/web/metadata/utils_test.exs index 85ef6033a..3daf852fb 100644 --- a/test/pleroma/web/metadata/utils_test.exs +++ b/test/pleroma/web/metadata/utils_test.exs @@ -72,7 +72,7 @@ defmodule Pleroma.Web.Metadata.UtilsTest do end end - describe "scrub_html_and_truncate/2" do + describe "scrub_html_and_truncate/3" do test "it returns text without encode HTML" do assert Utils.scrub_html_and_truncate("Pleroma's really cool!") == "Pleroma's really cool!" end From 40f14fd31cb1a92b7f1b2a533a725c709fefbb3b Mon Sep 17 00:00:00 2001 From: tusooa Date: Sun, 12 Mar 2023 16:06:00 +0000 Subject: [PATCH 09/89] Merge branch 'remove-crypt' into 'develop' Remove crypt(3) support Closes #3030 and #3062 See merge request pleroma/pleroma!3847 --- CHANGELOG.md | 1 + lib/pleroma/web/plugs/authentication_plug.ex | 8 ----- mix.exs | 1 - mix.lock | 1 - .../web/plugs/authentication_plug_test.exs | 30 ------------------- 5 files changed, 1 insertion(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a7ec1032..1742ee2f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed ### Removed +- BREAKING: Support for passwords generated with `crypt(3)` (Gnu Social migration artifact) ## 2.5.1 diff --git a/lib/pleroma/web/plugs/authentication_plug.ex b/lib/pleroma/web/plugs/authentication_plug.ex index a7fd697b5..f912a1542 100644 --- a/lib/pleroma/web/plugs/authentication_plug.ex +++ b/lib/pleroma/web/plugs/authentication_plug.ex @@ -38,10 +38,6 @@ defmodule Pleroma.Web.Plugs.AuthenticationPlug do def call(conn, _), do: conn - def checkpw(password, "$6" <> _ = password_hash) do - :crypt.crypt(password, password_hash) == password_hash - end - def checkpw(password, "$2" <> _ = password_hash) do # Handle bcrypt passwords for Mastodon migration Bcrypt.verify_pass(password, password_hash) @@ -60,10 +56,6 @@ defmodule Pleroma.Web.Plugs.AuthenticationPlug do do_update_password(user, password) end - def maybe_update_password(%User{password_hash: "$6" <> _} = user, password) do - do_update_password(user, password) - end - def maybe_update_password(user, _), do: {:ok, user} defp do_update_password(user, password) do diff --git a/mix.exs b/mix.exs index ab0be4deb..7439bbca6 100644 --- a/mix.exs +++ b/mix.exs @@ -150,7 +150,6 @@ defmodule Pleroma.Mixfile do {:sweet_xml, "~> 0.7.2"}, {:earmark, "~> 1.4.22"}, {:bbcode_pleroma, "~> 0.2.0"}, - {:crypt, "~> 1.0"}, {:cors_plug, "~> 2.0"}, {:web_push_encryption, "~> 0.3.1"}, {:swoosh, "~> 1.0"}, diff --git a/mix.lock b/mix.lock index 302786326..8419dc739 100644 --- a/mix.lock +++ b/mix.lock @@ -21,7 +21,6 @@ "cowlib": {:hex, :cowlib, "2.11.0", "0b9ff9c346629256c42ebe1eeb769a83c6cb771a6ee5960bd110ab0b9b872063", [:make, :rebar3], [], "hexpm", "2b3e9da0b21c4565751a6d4901c20d1b4cc25cbb7fd50d91d2ab6dd287bc86a9"}, "credo": {:hex, :credo, "1.6.7", "323f5734350fd23a456f2688b9430e7d517afb313fbd38671b8a4449798a7854", [:mix], [{:bunt, "~> 0.2.1", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "41e110bfb007f7eda7f897c10bf019ceab9a0b269ce79f015d54b0dcf4fc7dd3"}, "crontab": {:hex, :crontab, "1.1.8", "2ce0e74777dfcadb28a1debbea707e58b879e6aa0ffbf9c9bb540887bce43617", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, - "crypt": {:hex, :crypt, "1.0.1", "a3567e1c651a2ec42c6650d9f3ab789e0f12a508c060653a9bbb5fafe60f043c", [:rebar3], [], "hexpm", "968dffe321c7a5d9f9b4577c4a4ff56a1c26d1a8a2270eb22c7636a0b43d3982"}, "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"}, "db_connection": {:hex, :db_connection, "2.4.2", "f92e79aff2375299a16bcb069a14ee8615c3414863a6fef93156aee8e86c2ff3", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4fe53ca91b99f55ea249693a0229356a08f4d1a7931d8ffa79289b145fe83668"}, "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, diff --git a/test/pleroma/web/plugs/authentication_plug_test.exs b/test/pleroma/web/plugs/authentication_plug_test.exs index 41fdb93bc..b8acd01c5 100644 --- a/test/pleroma/web/plugs/authentication_plug_test.exs +++ b/test/pleroma/web/plugs/authentication_plug_test.exs @@ -70,28 +70,6 @@ defmodule Pleroma.Web.Plugs.AuthenticationPlugTest do assert "$pbkdf2" <> _ = user.password_hash end - @tag :skip_on_mac - test "with a crypt hash, it updates to a pkbdf2 hash", %{conn: conn} do - user = - insert(:user, - password_hash: - "$6$9psBWV8gxkGOZWBz$PmfCycChoxeJ3GgGzwvhlgacb9mUoZ.KUXNCssekER4SJ7bOK53uXrHNb2e4i8yPFgSKyzaW9CcmrDXWIEMtD1" - ) - - conn = - conn - |> assign(:auth_user, user) - |> assign(:auth_credentials, %{password: "password"}) - |> AuthenticationPlug.call(%{}) - - assert conn.assigns.user.id == conn.assigns.auth_user.id - assert conn.assigns.token == nil - assert PlugHelper.plug_skipped?(conn, OAuthScopesPlug) - - user = User.get_by_id(user.id) - assert "$pbkdf2" <> _ = user.password_hash - end - describe "checkpw/2" do test "check pbkdf2 hash" do hash = @@ -101,14 +79,6 @@ defmodule Pleroma.Web.Plugs.AuthenticationPlugTest do refute AuthenticationPlug.checkpw("test-password1", hash) end - @tag :skip_on_mac - test "check sha512-crypt hash" do - hash = - "$6$9psBWV8gxkGOZWBz$PmfCycChoxeJ3GgGzwvhlgacb9mUoZ.KUXNCssekER4SJ7bOK53uXrHNb2e4i8yPFgSKyzaW9CcmrDXWIEMtD1" - - assert AuthenticationPlug.checkpw("password", hash) - end - test "check bcrypt hash" do hash = "$2a$10$uyhC/R/zoE1ndwwCtMusK.TLVzkQ/Ugsbqp3uXI.CTTz0gBw.24jS" From ad38cc3b0c7895554291e874dfcfcf011b21f886 Mon Sep 17 00:00:00 2001 From: tusooa Date: Thu, 16 Mar 2023 08:11:44 +0000 Subject: [PATCH 10/89] Merge branch 'docs-otp-support' into 'develop' docs: Be more explicit about the level of compatibility of OTP releases See merge request pleroma/pleroma!3849 --- docs/installation/otp_en.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index 8c02201e6..f2812346b 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -2,15 +2,16 @@ {! backend/installation/otp_vs_from_source.include !} -This guide covers a installation using an OTP release. To install Pleroma from source, please check out the corresponding guide for your distro. +This guide covers a installation using OTP releases as built by the Pleroma project, it is meant as a fallback to distribution packages/recipes which are the preferred installation method. +To install Pleroma from source, please check out the corresponding guide for your distro. ## Pre-requisites -* A machine running Linux with GNU (e.g. Debian, Ubuntu) or musl (e.g. Alpine) libc and `x86_64`, `aarch64` or `armv7l` CPU, you have root access to. If you are not sure if it's compatible see [Detecting flavour section](#detecting-flavour) below +* A machine you have root access to running Debian GNU/Linux or compatible (eg. Ubuntu), or Alpine on `x86_64`, `aarch64` or `armv7l` CPU. If you are not sure what you are running see [Detecting flavour section](#detecting-flavour) below * A (sub)domain pointed to the machine -You will be running commands as root. If you aren't root already, please elevate your privileges by executing `sudo su`/`su`. +You will be running commands as root. If you aren't root already, please elevate your privileges by executing `sudo -i`/`su`. -While in theory OTP releases are possbile to install on any compatible machine, for the sake of simplicity this guide focuses only on Debian/Ubuntu and Alpine. +Similarly to other binaries, OTP releases tend to be only compatible with the distro they are built on, as such this guide focuses only on Debian/Ubuntu and Alpine. ### Detecting flavour @@ -19,7 +20,7 @@ Paste the following into the shell: arch="$(uname -m)";if [ "$arch" = "x86_64" ];then arch="amd64";elif [ "$arch" = "armv7l" ];then arch="arm";elif [ "$arch" = "aarch64" ];then arch="arm64";else echo "Unsupported arch: $arch">&2;fi;if getconf GNU_LIBC_VERSION>/dev/null;then libc_postfix="";elif [ "$(ldd 2>&1|head -c 9)" = "musl libc" ];then libc_postfix="-musl";elif [ "$(find /lib/libc.musl*|wc -l)" ];then libc_postfix="-musl";else echo "Unsupported libc">&2;fi;echo "$arch$libc_postfix" ``` -If your platform is supported the output will contain the flavour string, you will need it later. If not, this just means that we don't build releases for your platform, you can still try installing from source. +This should give your flavour string. If not this just means that we don't build releases for your platform, you can still try installing from source. ### Installing the required packages From e4288df50266a73ae1485c684b18d6f20f3234b0 Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Sat, 25 Mar 2023 05:40:35 +0000 Subject: [PATCH 11/89] Merge branch 'background-timeout' into 'develop' Set background worker timeout to 15 minutes See merge request pleroma/pleroma!3857 --- lib/pleroma/workers/background_worker.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index 3805293bc..794417612 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -45,5 +45,5 @@ defmodule Pleroma.Workers.BackgroundWorker do end @impl Oban.Worker - def timeout(_job), do: :timer.seconds(5) + def timeout(_job), do: :timer.seconds(900) end From 6a63dced4a567e0ba84c3c1e796eff93ba015255 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 30 Mar 2023 19:25:35 +0300 Subject: [PATCH 12/89] Fix tests for frontend installation --- lib/pleroma/web/api_spec/operations/admin/frontend_operation.ex | 2 +- .../web/admin_api/controllers/frontend_controller_test.exs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/operations/admin/frontend_operation.ex b/lib/pleroma/web/api_spec/operations/admin/frontend_operation.ex index 89b1c474c..3e85c44d2 100644 --- a/lib/pleroma/web/api_spec/operations/admin/frontend_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/frontend_operation.ex @@ -51,7 +51,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.FrontendOperation do name: %Schema{type: :string}, git: %Schema{type: :string, format: :uri, nullable: true}, build_url: %Schema{type: :string, format: :uri, nullable: true}, - ref: %Schema{type: :string}, + ref: %Schema{type: :string, nullable: true}, installed: %Schema{type: :boolean}, installed_refs: %Schema{type: :array, items: %Schema{type: :string}} } diff --git a/test/pleroma/web/admin_api/controllers/frontend_controller_test.exs b/test/pleroma/web/admin_api/controllers/frontend_controller_test.exs index 38a23b224..0d1a4999e 100644 --- a/test/pleroma/web/admin_api/controllers/frontend_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/frontend_controller_test.exs @@ -89,6 +89,7 @@ defmodule Pleroma.Web.AdminAPI.FrontendControllerTest do "build_url" => "http://gensokyo.2hu/builds/${ref}", "git" => nil, "installed" => true, + "installed_refs" => ["fantasy"], "name" => "pleroma", "ref" => "fantasy" } From 7997ba0abed74e45c54d10b634f283d0016a447f Mon Sep 17 00:00:00 2001 From: tusooa Date: Wed, 12 Apr 2023 11:49:33 -0400 Subject: [PATCH 13/89] Build images with kaniko --- .gitlab-ci.yml | 176 +++++++++++++++++++++++++------------------------ 1 file changed, 91 insertions(+), 85 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2e4897aa0..920463fd4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -373,104 +373,110 @@ arm64-musl: before_script: *before-release-musl script: *release -docker: +.kaniko: stage: docker - image: docker:latest + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] cache: {} dependencies: [] - variables: &docker-variables - DOCKER_DRIVER: overlay2 - DOCKER_HOST: unix:///var/run/docker.sock - IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA - IMAGE_TAG_SLUG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG - IMAGE_TAG_LATEST: $CI_REGISTRY_IMAGE:latest - IMAGE_TAG_LATEST_STABLE: $CI_REGISTRY_IMAGE:latest-stable - DOCKER_BUILDX_URL: https://github.com/docker/buildx/releases/download/v0.6.3/buildx-v0.6.3.linux-amd64 - DOCKER_BUILDX_HASH: 980e6b9655f971991fbbb5fd6cd19f1672386195 - before_script: &before-docker - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - docker pull $IMAGE_TAG_SLUG || true + before_script: &before-kaniko - export CI_JOB_TIMESTAMP=$(date --utc -Iseconds) - export CI_VCS_REF=$CI_COMMIT_SHORT_SHA - allow_failure: true - script: - - mkdir -p /root/.docker/cli-plugins - - wget "${DOCKER_BUILDX_URL}" -O ~/.docker/cli-plugins/docker-buildx - - echo "${DOCKER_BUILDX_HASH} /root/.docker/cli-plugins/docker-buildx" | sha1sum -c - - chmod +x ~/.docker/cli-plugins/docker-buildx - - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - docker buildx create --name mbuilder --driver docker-container --use - - docker buildx inspect --bootstrap - - docker buildx build --platform linux/amd64,linux/arm/v7,linux/arm64/v8 --push --cache-from $IMAGE_TAG_SLUG --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP -t $IMAGE_TAG -t $IMAGE_TAG_SLUG -t $IMAGE_TAG_LATEST . - tags: - - dind + - export IMAGE_TAG=$CI_REGISTRY_IMAGE/$BUILD_ARCH:$CI_COMMIT_SHORT_SHA + - export IMAGE_TAG_SLUG=$CI_REGISTRY_IMAGE/$BUILD_ARCH:$CI_COMMIT_REF_SLUG + - export IMAGE_TAG_LATEST=$CI_REGISTRY_IMAGE/$BUILD_ARCH:latest + - export IMAGE_TAG_LATEST_STABLE=$CI_REGISTRY_IMAGE/$BUILD_ARCH:latest-stable + - mkdir -p /kaniko/.docker + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json + +.kaniko-latest: + extends: .kaniko only: - develop@pleroma/pleroma - -docker-stable: - stage: docker - image: docker:latest - cache: {} - dependencies: [] - variables: *docker-variables - before_script: *before-docker - allow_failure: true script: - - mkdir -p /root/.docker/cli-plugins - - wget "${DOCKER_BUILDX_URL}" -O ~/.docker/cli-plugins/docker-buildx - - echo "${DOCKER_BUILDX_HASH} /root/.docker/cli-plugins/docker-buildx" | sha1sum -c - - chmod +x ~/.docker/cli-plugins/docker-buildx - - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - docker buildx create --name mbuilder --driver docker-container --use - - docker buildx inspect --bootstrap - - docker buildx build --platform linux/amd64,linux/arm/v7,linux/arm64/v8 --push --cache-from $IMAGE_TAG_SLUG --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP -t $IMAGE_TAG -t $IMAGE_TAG_SLUG -t $IMAGE_TAG_LATEST_STABLE . - tags: - - dind + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --customPlatform=$BUILD_ARCH --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP --destination $IMAGE_TAG --destination $IMAGE_TAG_SLUG --destination $IMAGE_TAG_LATEST + +.kaniko-stable: + extends: .kaniko only: - stable@pleroma/pleroma + script: + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --customPlatform=$BUILD_ARCH --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP --destination $IMAGE_TAG --destination $IMAGE_TAG_SLUG --destination $IMAGE_TAG_LATEST_STABLE -docker-release: - stage: docker - image: docker:latest - cache: {} - dependencies: [] - variables: *docker-variables - before_script: *before-docker - allow_failure: true - script: - script: - - mkdir -p /root/.docker/cli-plugins - - wget "${DOCKER_BUILDX_URL}" -O ~/.docker/cli-plugins/docker-buildx - - echo "${DOCKER_BUILDX_HASH} /root/.docker/cli-plugins/docker-buildx" | sha1sum -c - - chmod +x ~/.docker/cli-plugins/docker-buildx - - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - docker buildx create --name mbuilder --driver docker-container --use - - docker buildx inspect --bootstrap - - docker buildx build --platform linux/amd64,linux/arm/v7,linux/arm64/v8 --push --cache-from $IMAGE_TAG_SLUG --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP -t $IMAGE_TAG -t $IMAGE_TAG_SLUG . - tags: - - dind +.kaniko-release: + extends: .kaniko only: - /^release/.*$/@pleroma/pleroma + script: + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --customPlatform=$BUILD_ARCH --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP --destination $IMAGE_TAG --destination $IMAGE_TAG_SLUG -docker-adhoc: - stage: docker - image: docker:latest - cache: {} - dependencies: [] - variables: *docker-variables - before_script: *before-docker - allow_failure: true - script: - script: - - mkdir -p /root/.docker/cli-plugins - - wget "${DOCKER_BUILDX_URL}" -O ~/.docker/cli-plugins/docker-buildx - - echo "${DOCKER_BUILDX_HASH} /root/.docker/cli-plugins/docker-buildx" | sha1sum -c - - chmod +x ~/.docker/cli-plugins/docker-buildx - - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - docker buildx create --name mbuilder --driver docker-container --use - - docker buildx inspect --bootstrap - - docker buildx build --platform linux/amd64,linux/arm/v7,linux/arm64/v8 --push --cache-from $IMAGE_TAG_SLUG --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP -t $IMAGE_TAG -t $IMAGE_TAG_SLUG . - tags: - - dind +.kaniko-adhoc: + extends: .kaniko only: - /^build-docker/.*$/@pleroma/pleroma + script: + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --customPlatform=$BUILD_ARCH --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP --destination $IMAGE_TAG --destination $IMAGE_TAG_SLUG + +.kaniko:linux/amd64: + variables: + BUILD_ARCH: linux/amd64 + tags: + - amd64 + +.kaniko:linux/arm64: + variables: + BUILD_ARCH: linux/arm64 + tags: + - arm + +.kaniko:linux/arm: + variables: + BUILD_ARCH: linux/arm + tags: + - arm32-specified + +kaniko-latest:linux/amd64: + extends: + - .kaniko-latest + - .kaniko:linux/amd64 + +kaniko-latest:linux/arm64: + extends: + - .kaniko-latest + - .kaniko:linux/arm64 + +kaniko-latest:linux/arm: + extends: + - .kaniko-latest + - .kaniko:linux/arm + +kaniko-stable:linux/amd64: + extends: + - .kaniko-stable + - .kaniko:linux/amd64 + +kaniko-stable:linux/arm64: + extends: + - .kaniko-stable + - .kaniko:linux/arm64 + +kaniko-stable:linux/arm: + extends: + - .kaniko-stable + - .kaniko:linux/arm + +kaniko-release:linux/amd64: + extends: + - .kaniko-release + - .kaniko:linux/amd64 + +kaniko-release:linux/arm64: + extends: + - .kaniko-release + - .kaniko:linux/arm64 + +kaniko-release:linux/arm: + extends: + - .kaniko-release + - .kaniko:linux/arm From b37a90caa3ce01930a527e8f9d9b5e877efbb73b Mon Sep 17 00:00:00 2001 From: tusooa Date: Wed, 12 Apr 2023 12:38:47 -0400 Subject: [PATCH 14/89] Combine images of different platforms into one --- .gitlab-ci.yml | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 920463fd4..af882083e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,6 +31,7 @@ stages: - deploy - release - docker + - docker-combine before_script: - echo $MIX_ENV @@ -380,6 +381,14 @@ arm64-musl: entrypoint: [""] cache: {} dependencies: [] + needs: + - spec-build + - unit-testing + - unit-testing-erratic + - unit-testing-rum + - lint + - analysis + - cycles before_script: &before-kaniko - export CI_JOB_TIMESTAMP=$(date --utc -Iseconds) - export CI_VCS_REF=$CI_COMMIT_SHORT_SHA @@ -480,3 +489,65 @@ kaniko-release:linux/arm: extends: - .kaniko-release - .kaniko:linux/arm + +.docker-combine: + stage: docker-combine + image: docker:cli + cache: {} + before_script: + - 'BUILD_ARCHES="linux/amd64 linux/arm64/v8 linux/arm/v7"' + - export IMAGE_TAG=$CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA + - export IMAGE_TAG_SLUG=$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + - export IMAGE_TAG_LATEST=$CI_REGISTRY_IMAGE:latest + - export IMAGE_TAG_LATEST_STABLE=$CI_REGISTRY_IMAGE:latest-stable + - 'IMAGES=; for arch in $BUILD_ARCHES; do IMAGES="$IMAGES $CI_REGISTRY_IMAGE/$arch:$CI_COMMIT_SHORT_SHA"; done' + - 'IMAGES_SLUG=; for arch in $BUILD_ARCHES; do IMAGES_SLUG="$IMAGES_SLUG $CI_REGISTRY_IMAGE/$arch:$CI_COMMIT_REF_SLUG"; done' + - 'IMAGES_LATEST=; for arch in $BUILD_ARCHES; do IMAGES_LATEST="$IMAGES_LATEST $CI_REGISTRY_IMAGE/$arch:latest"; done' + - 'IMAGES_LATEST_STABLE=; for arch in $BUILD_ARCHES; do IMAGES_LATEST_STABLE="$IMAGES_LATEST_STABLE $CI_REGISTRY_IMAGE/$arch:latest"; done' + - echo "$CI_REGISTRY_PASSWORD" | docker login $CI_REGISTRY --username $CI_REGISTRY_USER --password-stdin + +docker-combine:latest: + extends: .docker-combine + only: + - develop@pleroma/pleroma + needs: + - 'kaniko-latest:linux/amd64' + - 'kaniko-latest:linux/arm64' + - 'kaniko-latest:linux/arm' + script: + - 'docker manifest create $IMAGE_TAG $IMAGES' + - 'docker manifest push $IMAGE_TAG' + - 'docker manifest create $IMAGE_TAG_SLUG $IMAGES_SLUG' + - 'docker manifest push $IMAGE_TAG_SLUG' + - 'docker manifest create $IMAGE_TAG_LATEST $IMAGES_LATEST' + - 'docker manifest push $IMAGE_TAG_LATEST' + +docker-combine:stable: + extends: .docker-combine + only: + - stable@pleroma/pleroma + needs: + - 'kaniko-stable:linux/amd64' + - 'kaniko-stable:linux/arm64' + - 'kaniko-stable:linux/arm' + script: + - 'docker manifest create $IMAGE_TAG $IMAGES' + - 'docker manifest push $IMAGE_TAG' + - 'docker manifest create $IMAGE_TAG_SLUG $IMAGES_SLUG' + - 'docker manifest push $IMAGE_TAG_SLUG' + - 'docker manifest create $IMAGE_TAG_LATEST_STABLE $IMAGES_LATEST_STABLE' + - 'docker manifest push $IMAGE_TAG_LATEST_STABLE' + +docker-combine:release: + extends: .docker-combine + only: + - /^release/.*$/@pleroma/pleroma + needs: + - 'kaniko-release:linux/amd64' + - 'kaniko-release:linux/arm64' + - 'kaniko-release:linux/arm' + script: + - 'docker manifest create $IMAGE_TAG $IMAGES' + - 'docker manifest push $IMAGE_TAG' + - 'docker manifest create $IMAGE_TAG_SLUG $IMAGES_SLUG' + - 'docker manifest push $IMAGE_TAG_SLUG' From 23bca0c4b3b19a6f82158bb35743cf035f497ef4 Mon Sep 17 00:00:00 2001 From: tusooa Date: Wed, 12 Apr 2023 12:40:26 -0400 Subject: [PATCH 15/89] Skip changelog entry --- changelog.d/3870.skip | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 changelog.d/3870.skip diff --git a/changelog.d/3870.skip b/changelog.d/3870.skip new file mode 100644 index 000000000..e69de29bb From 8f0f58e28b58cd9a4a05f1b927aa24fd63f9a3bd Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 17 Apr 2023 21:07:08 +0200 Subject: [PATCH 16/89] UploadedMedia: Add missing disposition_type to Content-Disposition Set it to `inline` because the vast majority of what's sent is multimedia content while `attachment` would have the side-effect of triggering a download dialog. Closes: https://git.pleroma.social/pleroma/pleroma/-/issues/3114 --- changelog.d/3873.fix | 1 + lib/pleroma/web/plugs/uploaded_media.ex | 2 +- test/pleroma/web/plugs/uploaded_media_plug_test.exs | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/3873.fix diff --git a/changelog.d/3873.fix b/changelog.d/3873.fix new file mode 100644 index 000000000..4699f7b58 --- /dev/null +++ b/changelog.d/3873.fix @@ -0,0 +1 @@ +UploadedMedia: Add missing disposition_type to Content-Disposition \ No newline at end of file diff --git a/lib/pleroma/web/plugs/uploaded_media.ex b/lib/pleroma/web/plugs/uploaded_media.ex index ad8143234..71d8f2f74 100644 --- a/lib/pleroma/web/plugs/uploaded_media.ex +++ b/lib/pleroma/web/plugs/uploaded_media.ex @@ -37,7 +37,7 @@ defmodule Pleroma.Web.Plugs.UploadedMedia do %{query_params: %{"name" => name}} = conn -> name = String.replace(name, "\"", "\\\"") - put_resp_header(conn, "content-disposition", "filename=\"#{name}\"") + put_resp_header(conn, "content-disposition", "inline; filename=\"#{name}\"") conn -> conn diff --git a/test/pleroma/web/plugs/uploaded_media_plug_test.exs b/test/pleroma/web/plugs/uploaded_media_plug_test.exs index ec46b0537..99ba49d37 100644 --- a/test/pleroma/web/plugs/uploaded_media_plug_test.exs +++ b/test/pleroma/web/plugs/uploaded_media_plug_test.exs @@ -37,7 +37,7 @@ defmodule Pleroma.Web.Plugs.UploadedMediaPlugTest do assert Enum.any?( conn.resp_headers, - &(&1 == {"content-disposition", "filename=\"\\\"cofe\\\".gif\""}) + &(&1 == {"content-disposition", "inline; filename=\"\\\"cofe\\\".gif\""}) ) end end From 2148ef5e2fcced6a52f0cfda38abcf697698dd1a Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 18 Apr 2023 00:07:39 +0200 Subject: [PATCH 17/89] UploadedMedia: Increase readability via ~s sigil --- lib/pleroma/web/plugs/uploaded_media.ex | 4 ++-- test/pleroma/web/plugs/uploaded_media_plug_test.exs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/plugs/uploaded_media.ex b/lib/pleroma/web/plugs/uploaded_media.ex index 71d8f2f74..8b3bc9acb 100644 --- a/lib/pleroma/web/plugs/uploaded_media.ex +++ b/lib/pleroma/web/plugs/uploaded_media.ex @@ -35,9 +35,9 @@ defmodule Pleroma.Web.Plugs.UploadedMedia do conn = case fetch_query_params(conn) do %{query_params: %{"name" => name}} = conn -> - name = String.replace(name, "\"", "\\\"") + name = String.replace(name, ~s["], ~s[\\"]) - put_resp_header(conn, "content-disposition", "inline; filename=\"#{name}\"") + put_resp_header(conn, "content-disposition", ~s[inline; filename="#{name}"]) conn -> conn diff --git a/test/pleroma/web/plugs/uploaded_media_plug_test.exs b/test/pleroma/web/plugs/uploaded_media_plug_test.exs index 99ba49d37..8323ff6ab 100644 --- a/test/pleroma/web/plugs/uploaded_media_plug_test.exs +++ b/test/pleroma/web/plugs/uploaded_media_plug_test.exs @@ -33,11 +33,11 @@ defmodule Pleroma.Web.Plugs.UploadedMediaPlugTest do test "sends Content-Disposition header when name param is set", %{ attachment_url: attachment_url } do - conn = get(build_conn(), attachment_url <> "?name=\"cofe\".gif") + conn = get(build_conn(), attachment_url <> ~s[?name="cofe".gif]) assert Enum.any?( conn.resp_headers, - &(&1 == {"content-disposition", "inline; filename=\"\\\"cofe\\\".gif\""}) + &(&1 == {"content-disposition", ~s[inline; filename="\\"cofe\\".gif"]}) ) end end From 50e237759a0aa378b32393ba57d50eaaff5029e0 Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 22 Apr 2023 20:30:17 -0400 Subject: [PATCH 18/89] Use git diff to search for changelog entry --- .gitlab-ci.yml | 1 + tools/check-changelog | 23 ++++++++--------------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2e4897aa0..6e498bf89 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -49,6 +49,7 @@ check-changelog: after_script: '' cache: {} script: + - apk add git - sh ./tools/check-changelog build: diff --git a/tools/check-changelog b/tools/check-changelog index b94b52755..0b57c4d1e 100644 --- a/tools/check-changelog +++ b/tools/check-changelog @@ -1,22 +1,15 @@ #!/bin/sh -echo "looking for change log of $CI_MERGE_REQUEST_IID" +echo "looking for change log" -count=0 -for i in add remove fix security skip; do - [ -f changelog.d/"$CI_MERGE_REQUEST_IID"."$i" ] - retcode=$? - if [ $retcode -eq 0 ]; then - echo "found $CI_MERGE_REQUEST_IID.$i" - count=$(( count + 1 )) - else - echo "no $CI_MERGE_REQUEST_IID.$i" - fi -done -if [ $count -gt 0 ]; then - echo "ok" +git diff --raw $CI_MERGE_REQUEST_TARGET_BRANCH_NAME HEAD -- changelog.d | \ + grep ' A\t' | grep '\.\(skip\|add\|remove\|fix\|security\)$' +ret=$? + +if [ $ret -eq 0 ]; then + echo "found a changelog entry" exit 0 else - echo "must have a changelog entry or explicitly skip it" + echo "changelog entry not found" exit 1 fi From e13b331762881933d66597b3145b4d20d6e51912 Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 22 Apr 2023 20:40:39 -0400 Subject: [PATCH 19/89] Fetch upstream in the repo --- tools/check-changelog | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/check-changelog b/tools/check-changelog index 0b57c4d1e..1409be9f0 100644 --- a/tools/check-changelog +++ b/tools/check-changelog @@ -2,7 +2,10 @@ echo "looking for change log" -git diff --raw $CI_MERGE_REQUEST_TARGET_BRANCH_NAME HEAD -- changelog.d | \ +git remote add upstream https://git.pleroma.social/pleroma/pleroma.git +git fetch upstream ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}:refs/remotes/upstream/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME + +git diff --raw upstream/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME HEAD -- changelog.d | \ grep ' A\t' | grep '\.\(skip\|add\|remove\|fix\|security\)$' ret=$? From c1aa83069daaa7df906f63b3417e14b7edbfae94 Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 22 Apr 2023 20:45:27 -0400 Subject: [PATCH 20/89] Skip changelog --- changelog.d/changelog-improve.skip | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 changelog.d/changelog-improve.skip diff --git a/changelog.d/changelog-improve.skip b/changelog.d/changelog-improve.skip new file mode 100644 index 000000000..e69de29bb From 30bc37c3cab8ebf7be667fb291f0d99c17d808ef Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 22 Apr 2023 21:02:13 -0400 Subject: [PATCH 21/89] Explain changelog.d in merge request templates --- .gitlab/merge_request_templates/Default.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .gitlab/merge_request_templates/Default.md diff --git a/.gitlab/merge_request_templates/Default.md b/.gitlab/merge_request_templates/Default.md new file mode 100644 index 000000000..fdf219f99 --- /dev/null +++ b/.gitlab/merge_request_templates/Default.md @@ -0,0 +1,10 @@ +### Checklist +- [ ] Adding a changelog: In the `changelog.d` directory, create a file named `.`. + + `` can be anything, but we recommend using a more or less unique identifier to avoid collisions, such as the branch name. + + `` can be `add`, `remove`, `fix`, `security` or `skip`. `skip` is only used if there is no user-visible change in the MR (for example, only editing comments in the code). Otherwise, choose a type that corresponds to your change. + + In the file, write the changelog entry. For example, if an MR adds group functionality, we can create a file named `group.add` and write `Add group functionality` in it. + + If one changelog entry is not enough, you may add more. But that might mean you can split it into two MRs. Only use more than one changelog entry if you really need to (for example, when one change in the code fix two different bugs, or when refactoring). From ae8f359f22d8d02c2937df0a81052fc3504d4248 Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 22 Apr 2023 21:07:18 -0400 Subject: [PATCH 22/89] Skip changelog check for automated MRs --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6e498bf89..2eee3bc68 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,6 +44,10 @@ check-changelog: stage: check-changelog image: alpine rules: + - if: $CI_MERGE_REQUEST_SOURCE_PROJECT_PATH == 'pleroma/pleroma' && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == 'weblate-extract' + when: never + - if: $CI_MERGE_REQUEST_SOURCE_PROJECT_PATH == 'pleroma/pleroma' && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == 'weblate' + when: never - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" before_script: '' after_script: '' From 0231a09310895bd2af24eff221de97eb8c92307e Mon Sep 17 00:00:00 2001 From: duponin Date: Sun, 16 Apr 2023 15:50:32 +0200 Subject: [PATCH 23/89] Remove SSH/BBS feature from core And link to sshocial, the replacement client for this removed feature --- config/config.exs | 3 - config/description.exs | 39 --- docs/configuration/cheatsheet.md | 17 +- lib/pleroma/bbs/authenticator.ex | 20 -- lib/pleroma/bbs/handler.ex | 246 ------------------ mix.exs | 4 +- mix.lock | 1 - test/pleroma/bbs/handler_test.exs | 89 ------- .../controllers/config_controller_test.exs | 6 +- 9 files changed, 4 insertions(+), 421 deletions(-) delete mode 100644 lib/pleroma/bbs/authenticator.ex delete mode 100644 lib/pleroma/bbs/handler.ex delete mode 100644 test/pleroma/bbs/handler_test.exs diff --git a/config/config.exs b/config/config.exs index e41ec2f91..178b6be99 100644 --- a/config/config.exs +++ b/config/config.exs @@ -617,9 +617,6 @@ config :pleroma, :ldap, base: System.get_env("LDAP_BASE") || "dc=example,dc=com", uid: System.get_env("LDAP_UID") || "cn" -config :esshd, - enabled: false - oauth_consumer_strategies = System.get_env("OAUTH_CONSUMER_STRATEGIES") |> to_string() diff --git a/config/description.exs b/config/description.exs index 78dc8770d..5f8add8a4 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2628,45 +2628,6 @@ config :pleroma, :config_description, [ } ] }, - %{ - group: :esshd, - label: "ESSHD", - type: :group, - description: - "Before enabling this you must add :esshd to mix.exs as one of the extra_applications " <> - "and generate host keys in your priv dir with ssh-keygen -m PEM -N \"\" -b 2048 -t rsa -f ssh_host_rsa_key", - children: [ - %{ - key: :enabled, - type: :boolean, - description: "Enables SSH" - }, - %{ - key: :priv_dir, - type: :string, - description: "Dir with SSH keys", - suggestions: ["/some/path/ssh_keys"] - }, - %{ - key: :handler, - type: :string, - description: "Handler module", - suggestions: ["Pleroma.BBS.Handler"] - }, - %{ - key: :port, - type: :integer, - description: "Port to connect", - suggestions: [10_022] - }, - %{ - key: :password_authenticator, - type: :string, - description: "Authenticator module", - suggestions: ["Pleroma.BBS.Authenticator"] - } - ] - }, %{ group: :mime, label: "Mime Types", diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index bbdf30a0f..70abe3e0c 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -873,21 +873,8 @@ This will probably take a long time. ### BBS / SSH access -To enable simple command line interface accessible over ssh, add a setting like this to your configuration file: - -```exs -app_dir = File.cwd! -priv_dir = Path.join([app_dir, "priv/ssh_keys"]) - -config :esshd, - enabled: true, - priv_dir: priv_dir, - handler: "Pleroma.BBS.Handler", - port: 10_022, - password_authenticator: "Pleroma.BBS.Authenticator" -``` - -Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -m PEM -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT` +This feature has been removed from Pleroma core. +However, a client has been made and is available at https://git.pleroma.social/Duponin/sshocial. ### :gopher * `enabled`: Enables the gopher interface diff --git a/lib/pleroma/bbs/authenticator.ex b/lib/pleroma/bbs/authenticator.ex deleted file mode 100644 index 0f7543ff5..000000000 --- a/lib/pleroma/bbs/authenticator.ex +++ /dev/null @@ -1,20 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.BBS.Authenticator do - use Sshd.PasswordAuthenticator - alias Pleroma.User - alias Pleroma.Web.Plugs.AuthenticationPlug - - def authenticate(username, password) do - username = to_string(username) - password = to_string(password) - - with %User{} = user <- User.get_by_nickname(username) do - AuthenticationPlug.checkpw(password, user.password_hash) - else - _e -> false - end - end -end diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex deleted file mode 100644 index 27799338f..000000000 --- a/lib/pleroma/bbs/handler.ex +++ /dev/null @@ -1,246 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.BBS.Handler do - use Sshd.ShellHandler - alias Pleroma.Activity - alias Pleroma.HTML - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.CommonAPI - - def on_shell(username, _pubkey, _ip, _port) do - :ok = IO.puts("Welcome to #{Pleroma.Config.get([:instance, :name])}!") - user = Pleroma.User.get_cached_by_nickname(to_string(username)) - Logger.debug("#{inspect(user)}") - loop(run_state(user: user)) - end - - def on_connect(username, ip, port, method) do - Logger.debug(fn -> - """ - Incoming SSH shell #{inspect(self())} requested for #{username} from #{inspect(ip)}:#{inspect(port)} using #{inspect(method)} - """ - end) - end - - def on_disconnect(username, ip, port) do - Logger.debug(fn -> - "Disconnecting SSH shell for #{username} from #{inspect(ip)}:#{inspect(port)}" - end) - end - - defp loop(state) do - self_pid = self() - counter = state.counter - prefix = state.prefix - user = state.user - - input = spawn(fn -> io_get(self_pid, prefix, counter, user.nickname) end) - wait_input(state, input) - end - - def puts_activity(activity) do - status = Pleroma.Web.MastodonAPI.StatusView.render("show.json", %{activity: activity}) - - IO.puts("-- #{status.id} by #{status.account.display_name} (#{status.account.acct})") - - status.content - |> String.split("
") - |> Enum.map(&HTML.strip_tags/1) - |> Enum.map(&HtmlEntities.decode/1) - |> Enum.map(&IO.puts/1) - end - - def puts_notification(activity, user) do - notification = - Pleroma.Web.MastodonAPI.NotificationView.render("show.json", %{ - notification: activity, - for: user - }) - - IO.puts( - "== (#{notification.type}) #{notification.status.id} by #{notification.account.display_name} (#{notification.account.acct})" - ) - - notification.status.content - |> String.split("
") - |> Enum.map(&HTML.strip_tags/1) - |> Enum.map(&HtmlEntities.decode/1) - |> (fn x -> - case x do - [content] -> - "> " <> content - - [head | _tail] -> - # "> " <> hd <> "..." - head - |> String.slice(1, 80) - |> (fn x -> "> " <> x <> "..." end).() - end - end).() - |> IO.puts() - - IO.puts("") - end - - def handle_command(state, "help") do - IO.puts("Available commands:") - IO.puts("help - This help") - IO.puts("home - Show the home timeline") - IO.puts("p - Post the given text") - IO.puts("r - Reply to the post with the given id") - IO.puts("t - Show a thread from the given id") - IO.puts("n - Show notifications") - IO.puts("n read - Mark all notifactions as read") - IO.puts("f - Favourites the post with the given id") - IO.puts("R - Repeat the post with the given id") - IO.puts("quit - Quit") - - state - end - - def handle_command(%{user: user} = state, "r " <> text) do - text = String.trim(text) - [activity_id, rest] = String.split(text, " ", parts: 2) - - with %Activity{} <- Activity.get_by_id(activity_id), - {:ok, _activity} <- - CommonAPI.post(user, %{status: rest, in_reply_to_status_id: activity_id}) do - IO.puts("Replied!") - else - _e -> IO.puts("Could not reply...") - end - - state - end - - def handle_command(%{user: user} = state, "t " <> activity_id) do - with %Activity{} = activity <- Activity.get_by_id(activity_id) do - activities = - ActivityPub.fetch_activities_for_context(activity.data["context"], %{ - blocking_user: user, - user: user, - exclude_id: activity.id - }) - - case activities do - [] -> - activity_id - |> Activity.get_by_id() - |> puts_activity() - - _ -> - activities - |> Enum.reverse() - |> Enum.each(&puts_activity/1) - end - else - _e -> IO.puts("Could not show this thread...") - end - - state - end - - def handle_command(%{user: user} = state, "n read") do - Pleroma.Notification.clear(user) - IO.puts("All notifications were marked as read") - - state - end - - def handle_command(%{user: user} = state, "n") do - user - |> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(%{}) - |> Enum.each(&puts_notification(&1, user)) - - state - end - - def handle_command(%{user: user} = state, "p " <> text) do - text = String.trim(text) - - with {:ok, activity} <- CommonAPI.post(user, %{status: text}) do - IO.puts("Posted! ID: #{activity.id}") - else - _e -> IO.puts("Could not post...") - end - - state - end - - def handle_command(%{user: user} = state, "f " <> id) do - id = String.trim(id) - - with %Activity{} = activity <- Activity.get_by_id(id), - {:ok, _activity} <- CommonAPI.favorite(user, activity) do - IO.puts("Favourited!") - else - _e -> IO.puts("Could not Favourite...") - end - - state - end - - def handle_command(state, "home") do - user = state.user - - params = - %{} - |> Map.put(:type, ["Create"]) - |> Map.put(:blocking_user, user) - |> Map.put(:muting_user, user) - |> Map.put(:user, user) - - activities = - [user.ap_id | Pleroma.User.following(user)] - |> ActivityPub.fetch_activities(params) - - Enum.each(activities, fn activity -> - puts_activity(activity) - end) - - state - end - - def handle_command(state, command) do - IO.puts("Unknown command '#{command}'") - state - end - - defp wait_input(state, input) do - receive do - {:input, ^input, "quit\n"} -> - IO.puts("Exiting...") - - {:input, ^input, code} when is_binary(code) -> - code = String.trim(code) - - state = handle_command(state, code) - - loop(%{state | counter: state.counter + 1}) - - {:input, ^input, {:error, :interrupted}} -> - IO.puts("Caught Ctrl+C...") - loop(%{state | counter: state.counter + 1}) - - {:input, ^input, msg} -> - :ok = Logger.warn("received unknown message: #{inspect(msg)}") - loop(%{state | counter: state.counter + 1}) - end - end - - defp run_state(opts) do - %{prefix: "pleroma", counter: 1, user: opts[:user]} - end - - defp io_get(pid, prefix, counter, username) do - prompt = prompt(prefix, counter, username) - send(pid, {:input, self(), IO.gets(:stdio, prompt)}) - end - - defp prompt(prefix, counter, username) do - prompt = "#{username}@#{prefix}:#{counter}>" - prompt <> " " - end -end diff --git a/mix.exs b/mix.exs index 027442e33..afce46d22 100644 --- a/mix.exs +++ b/mix.exs @@ -78,8 +78,7 @@ defmodule Pleroma.Mixfile do :comeonin, :fast_sanitize, :os_mon, - :ssl, - :esshd + :ssl ], included_applications: [:ex_syslogger] ] @@ -181,7 +180,6 @@ defmodule Pleroma.Mixfile do {:joken, "~> 2.0"}, {:benchee, "~> 1.0"}, {:pot, "~> 1.0"}, - {:esshd, "~> 0.1.0", runtime: Application.get_env(:esshd, :enabled, false)}, {:ex_const, "~> 0.2"}, {:plug_static_index_html, "~> 1.0.0"}, {:flake_id, "~> 0.1.0"}, diff --git a/mix.lock b/mix.lock index 0af2c4292..43012a58e 100644 --- a/mix.lock +++ b/mix.lock @@ -35,7 +35,6 @@ "eimp": {:hex, :eimp, "1.0.14", "fc297f0c7e2700457a95a60c7010a5f1dcb768a083b6d53f49cd94ab95a28f22", [:rebar3], [{:p1_utils, "1.0.18", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm", "501133f3112079b92d9e22da8b88bf4f0e13d4d67ae9c15c42c30bd25ceb83b6"}, "elixir_make": {:hex, :elixir_make, "0.6.3", "bc07d53221216838d79e03a8019d0839786703129599e9619f4ab74c8c096eac", [:mix], [], "hexpm", "f5cbd651c5678bcaabdbb7857658ee106b12509cd976c2c2fca99688e1daf716"}, "esbuild": {:hex, :esbuild, "0.5.0", "d5bb08ff049d7880ee3609ed5c4b864bd2f46445ea40b16b4acead724fb4c4a3", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "f183a0b332d963c4cfaf585477695ea59eef9a6f2204fdd0efa00e099694ffe5"}, - "esshd": {:hex, :esshd, "0.1.1", "d4dd4c46698093a40a56afecce8a46e246eb35463c457c246dacba2e056f31b5", [:mix], [], "hexpm", "d73e341e3009d390aa36387dc8862860bf9f874c94d9fd92ade2926376f49981"}, "eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"}, "ex_aws": {:hex, :ex_aws, "2.1.9", "dc4865ecc20a05190a34a0ac5213e3e5e2b0a75a0c2835e923ae7bfeac5e3c31", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "3e6c776703c9076001fbe1f7c049535f042cb2afa0d2cbd3b47cbc4e92ac0d10"}, "ex_aws_s3": {:hex, :ex_aws_s3, "2.4.0", "ce8decb6b523381812798396bc0e3aaa62282e1b40520125d1f4eff4abdff0f4", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "85dda6e27754d94582869d39cba3241d9ea60b6aa4167f9c88e309dc687e56bb"}, diff --git a/test/pleroma/bbs/handler_test.exs b/test/pleroma/bbs/handler_test.exs deleted file mode 100644 index aea3b6ead..000000000 --- a/test/pleroma/bbs/handler_test.exs +++ /dev/null @@ -1,89 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.BBS.HandlerTest do - use Pleroma.DataCase, async: true - alias Pleroma.Activity - alias Pleroma.BBS.Handler - alias Pleroma.Object - alias Pleroma.Repo - alias Pleroma.User - alias Pleroma.Web.CommonAPI - - import ExUnit.CaptureIO - import Pleroma.Factory - import Ecto.Query - - test "getting the home timeline" do - user = insert(:user) - followed = insert(:user) - - {:ok, user, followed} = User.follow(user, followed) - - {:ok, _first} = CommonAPI.post(user, %{status: "hey"}) - {:ok, _second} = CommonAPI.post(followed, %{status: "hello"}) - - output = - capture_io(fn -> - Handler.handle_command(%{user: user}, "home") - end) - - assert output =~ user.nickname - assert output =~ followed.nickname - - assert output =~ "hey" - assert output =~ "hello" - end - - test "posting" do - user = insert(:user) - - output = - capture_io(fn -> - Handler.handle_command(%{user: user}, "p this is a test post") - end) - - assert output =~ "Posted" - - activity = - Repo.one( - from(a in Activity, - where: fragment("?->>'type' = ?", a.data, "Create") - ) - ) - - assert activity.actor == user.ap_id - object = Object.normalize(activity, fetch: false) - assert object.data["content"] == "this is a test post" - end - - test "replying" do - user = insert(:user) - another_user = insert(:user) - - {:ok, activity} = CommonAPI.post(another_user, %{status: "this is a test post"}) - activity_object = Object.normalize(activity, fetch: false) - - output = - capture_io(fn -> - Handler.handle_command(%{user: user}, "r #{activity.id} this is a reply") - end) - - assert output =~ "Replied" - - reply = - Repo.one( - from(a in Activity, - where: fragment("?->>'type' = ?", a.data, "Create"), - where: a.actor == ^user.ap_id - ) - ) - - assert reply.actor == user.ap_id - - reply_object_data = Object.normalize(reply, fetch: false).data - assert reply_object_data["content"] == "this is a reply" - assert reply_object_data["inReplyTo"] == activity_object.data["id"] - end -end diff --git a/test/pleroma/web/admin_api/controllers/config_controller_test.exs b/test/pleroma/web/admin_api/controllers/config_controller_test.exs index 9fb5fb520..f13a9bd42 100644 --- a/test/pleroma/web/admin_api/controllers/config_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/config_controller_test.exs @@ -1502,8 +1502,7 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do clear_config(:database_config_whitelist, [ {:pleroma, :instance}, {:pleroma, :activitypub}, - {:pleroma, Pleroma.Upload}, - {:esshd} + {:pleroma, Pleroma.Upload} ]) conn = get(conn, "/api/pleroma/admin/config/descriptions") @@ -1522,9 +1521,6 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do web_endpoint = Enum.find(children, fn c -> c["key"] == "Pleroma.Upload" end) assert web_endpoint["children"] - - esshd = Enum.find(children, fn c -> c["group"] == ":esshd" end) - assert esshd["children"] end end end From b2dc9ad9d8bba1ddc213bc98b68b2d64b172733d Mon Sep 17 00:00:00 2001 From: duponin Date: Sun, 16 Apr 2023 16:03:49 +0200 Subject: [PATCH 24/89] fix test after removing esshd/SSH feature --- .../web/admin_api/controllers/config_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pleroma/web/admin_api/controllers/config_controller_test.exs b/test/pleroma/web/admin_api/controllers/config_controller_test.exs index f13a9bd42..19ce3681c 100644 --- a/test/pleroma/web/admin_api/controllers/config_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/config_controller_test.exs @@ -1509,7 +1509,7 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do children = json_response_and_validate_schema(conn, 200) - assert length(children) == 4 + assert length(children) == 3 assert Enum.count(children, fn c -> c["group"] == ":pleroma" end) == 3 From af38c6104ea0a5724d169dbe63efca7e158dbeea Mon Sep 17 00:00:00 2001 From: duponin Date: Sun, 23 Apr 2023 10:58:50 +0200 Subject: [PATCH 25/89] add changelog entry for BBS/SSH feature remove --- changelog.d/3872.remove | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3872.remove diff --git a/changelog.d/3872.remove b/changelog.d/3872.remove new file mode 100644 index 000000000..54cbb660e --- /dev/null +++ b/changelog.d/3872.remove @@ -0,0 +1 @@ +remove BBS/SSH feature, replaced by an external bridge. \ No newline at end of file From 66d23713e9fa16fc48931bb4efccd59d631e2610 Mon Sep 17 00:00:00 2001 From: tusooa Date: Mon, 24 Apr 2023 19:56:54 -0400 Subject: [PATCH 26/89] Do not use nested levels for arch --- .gitlab-ci.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index af882083e..8e971ec89 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -392,10 +392,10 @@ arm64-musl: before_script: &before-kaniko - export CI_JOB_TIMESTAMP=$(date --utc -Iseconds) - export CI_VCS_REF=$CI_COMMIT_SHORT_SHA - - export IMAGE_TAG=$CI_REGISTRY_IMAGE/$BUILD_ARCH:$CI_COMMIT_SHORT_SHA - - export IMAGE_TAG_SLUG=$CI_REGISTRY_IMAGE/$BUILD_ARCH:$CI_COMMIT_REF_SLUG - - export IMAGE_TAG_LATEST=$CI_REGISTRY_IMAGE/$BUILD_ARCH:latest - - export IMAGE_TAG_LATEST_STABLE=$CI_REGISTRY_IMAGE/$BUILD_ARCH:latest-stable + - export IMAGE_TAG=$CI_REGISTRY_IMAGE/$BUILD_ARCH_IMG_SUFFIX:$CI_COMMIT_SHORT_SHA + - export IMAGE_TAG_SLUG=$CI_REGISTRY_IMAGE/$BUILD_ARCH_IMG_SUFFIX:$CI_COMMIT_REF_SLUG + - export IMAGE_TAG_LATEST=$CI_REGISTRY_IMAGE/$BUILD_ARCH_IMG_SUFFIX:latest + - export IMAGE_TAG_LATEST_STABLE=$CI_REGISTRY_IMAGE/$BUILD_ARCH_IMG_SUFFIX:latest-stable - mkdir -p /kaniko/.docker - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json @@ -430,18 +430,21 @@ arm64-musl: .kaniko:linux/amd64: variables: BUILD_ARCH: linux/amd64 + BUILD_ARCH_IMG_SUFFIX: linux-amd64 tags: - amd64 .kaniko:linux/arm64: variables: - BUILD_ARCH: linux/arm64 + BUILD_ARCH: linux/arm64/v8 + BUILD_ARCH_IMG_SUFFIX: linux-arm64-v8 tags: - arm .kaniko:linux/arm: variables: - BUILD_ARCH: linux/arm + BUILD_ARCH: linux/arm/v7 + BUILD_ARCH_IMG_SUFFIX: linux-arm-v7 tags: - arm32-specified @@ -495,7 +498,7 @@ kaniko-release:linux/arm: image: docker:cli cache: {} before_script: - - 'BUILD_ARCHES="linux/amd64 linux/arm64/v8 linux/arm/v7"' + - 'BUILD_ARCHES="linux-amd64 linux-arm64-v8 linux-arm-v7"' - export IMAGE_TAG=$CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA - export IMAGE_TAG_SLUG=$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG - export IMAGE_TAG_LATEST=$CI_REGISTRY_IMAGE:latest From 2736c3f29406d11ed244c2e165f746f32f8b7273 Mon Sep 17 00:00:00 2001 From: tusooa Date: Mon, 24 Apr 2023 19:57:32 -0400 Subject: [PATCH 27/89] Use --custom-platform to replace the deprecated one --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8e971ec89..a98e3ee6b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -404,28 +404,28 @@ arm64-musl: only: - develop@pleroma/pleroma script: - - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --customPlatform=$BUILD_ARCH --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP --destination $IMAGE_TAG --destination $IMAGE_TAG_SLUG --destination $IMAGE_TAG_LATEST + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --custom-platform=$BUILD_ARCH --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP --destination $IMAGE_TAG --destination $IMAGE_TAG_SLUG --destination $IMAGE_TAG_LATEST .kaniko-stable: extends: .kaniko only: - stable@pleroma/pleroma script: - - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --customPlatform=$BUILD_ARCH --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP --destination $IMAGE_TAG --destination $IMAGE_TAG_SLUG --destination $IMAGE_TAG_LATEST_STABLE + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --custom-platform=$BUILD_ARCH --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP --destination $IMAGE_TAG --destination $IMAGE_TAG_SLUG --destination $IMAGE_TAG_LATEST_STABLE .kaniko-release: extends: .kaniko only: - /^release/.*$/@pleroma/pleroma script: - - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --customPlatform=$BUILD_ARCH --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP --destination $IMAGE_TAG --destination $IMAGE_TAG_SLUG + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --custom-platform=$BUILD_ARCH --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP --destination $IMAGE_TAG --destination $IMAGE_TAG_SLUG .kaniko-adhoc: extends: .kaniko only: - /^build-docker/.*$/@pleroma/pleroma script: - - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --customPlatform=$BUILD_ARCH --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP --destination $IMAGE_TAG --destination $IMAGE_TAG_SLUG + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --custom-platform=$BUILD_ARCH --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP --destination $IMAGE_TAG --destination $IMAGE_TAG_SLUG .kaniko:linux/amd64: variables: From c5ffdd060955bef1b86506c22dfd57eba6c64e41 Mon Sep 17 00:00:00 2001 From: tusooa Date: Mon, 24 Apr 2023 20:03:59 -0400 Subject: [PATCH 28/89] Use self-built elixir image for arm --- .gitlab-ci.yml | 9 ++++++--- Dockerfile | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a98e3ee6b..d0b2bd998 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -411,26 +411,27 @@ arm64-musl: only: - stable@pleroma/pleroma script: - - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --custom-platform=$BUILD_ARCH --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP --destination $IMAGE_TAG --destination $IMAGE_TAG_SLUG --destination $IMAGE_TAG_LATEST_STABLE + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --custom-platform=$BUILD_ARCH --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP --build-arg ELIXIR_IMG=$ELIXIR_IMG --destination $IMAGE_TAG --destination $IMAGE_TAG_SLUG --destination $IMAGE_TAG_LATEST_STABLE .kaniko-release: extends: .kaniko only: - /^release/.*$/@pleroma/pleroma script: - - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --custom-platform=$BUILD_ARCH --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP --destination $IMAGE_TAG --destination $IMAGE_TAG_SLUG + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --custom-platform=$BUILD_ARCH --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP --build-arg ELIXIR_IMG=$ELIXIR_IMG --destination $IMAGE_TAG --destination $IMAGE_TAG_SLUG .kaniko-adhoc: extends: .kaniko only: - /^build-docker/.*$/@pleroma/pleroma script: - - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --custom-platform=$BUILD_ARCH --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP --destination $IMAGE_TAG --destination $IMAGE_TAG_SLUG + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --custom-platform=$BUILD_ARCH --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP --build-arg ELIXIR_IMG=$ELIXIR_IMG --destination $IMAGE_TAG --destination $IMAGE_TAG_SLUG .kaniko:linux/amd64: variables: BUILD_ARCH: linux/amd64 BUILD_ARCH_IMG_SUFFIX: linux-amd64 + ELIXIR_IMG: hexpm/elixir tags: - amd64 @@ -438,6 +439,7 @@ arm64-musl: variables: BUILD_ARCH: linux/arm64/v8 BUILD_ARCH_IMG_SUFFIX: linux-arm64-v8 + ELIXIR_IMG: hexpm/elixir tags: - arm @@ -445,6 +447,7 @@ arm64-musl: variables: BUILD_ARCH: linux/arm/v7 BUILD_ARCH_IMG_SUFFIX: linux-arm-v7 + ELIXIR_IMG: git.pleroma.social:5050/pleroma/ci-image/elixir-linux-arm-v7 tags: - arm32-specified diff --git a/Dockerfile b/Dockerfile index 8c3ff3ac5..310d18104 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,9 @@ +ARG ELIXIR_IMG=hexpm/elixir ARG ELIXIR_VER=1.11.4 ARG ERLANG_VER=24.2.1 ARG ALPINE_VER=3.17.0 -FROM hexpm/elixir:${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER} as build +FROM ${ELIXIR_IMG}:${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER} as build COPY . . From 9c7b036401457ed801d03dae52bc0854bfa62715 Mon Sep 17 00:00:00 2001 From: tusooa Date: Tue, 25 Apr 2023 21:38:54 -0400 Subject: [PATCH 29/89] Add ELIXIR_IMG arg to latest --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d0b2bd998..154083c68 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -404,7 +404,7 @@ arm64-musl: only: - develop@pleroma/pleroma script: - - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --custom-platform=$BUILD_ARCH --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP --destination $IMAGE_TAG --destination $IMAGE_TAG_SLUG --destination $IMAGE_TAG_LATEST + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --custom-platform=$BUILD_ARCH --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP --build-arg ELIXIR_IMG=$ELIXIR_IMG --destination $IMAGE_TAG --destination $IMAGE_TAG_SLUG --destination $IMAGE_TAG_LATEST .kaniko-stable: extends: .kaniko From 47e95fe9f5732b26530045a4cfe81538249c64d3 Mon Sep 17 00:00:00 2001 From: tusooa Date: Tue, 25 Apr 2023 21:40:28 -0400 Subject: [PATCH 30/89] Add changelog for 3876 --- changelog.d/3876.skip | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 changelog.d/3876.skip diff --git a/changelog.d/3876.skip b/changelog.d/3876.skip new file mode 100644 index 000000000..e69de29bb From a94691720641c47fef9efdaf619abc6f82cdbf29 Mon Sep 17 00:00:00 2001 From: tusooa Date: Wed, 26 Apr 2023 07:18:39 -0400 Subject: [PATCH 31/89] Work around docker login needing daemon --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 154083c68..72ac4981d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -510,7 +510,8 @@ kaniko-release:linux/arm: - 'IMAGES_SLUG=; for arch in $BUILD_ARCHES; do IMAGES_SLUG="$IMAGES_SLUG $CI_REGISTRY_IMAGE/$arch:$CI_COMMIT_REF_SLUG"; done' - 'IMAGES_LATEST=; for arch in $BUILD_ARCHES; do IMAGES_LATEST="$IMAGES_LATEST $CI_REGISTRY_IMAGE/$arch:latest"; done' - 'IMAGES_LATEST_STABLE=; for arch in $BUILD_ARCHES; do IMAGES_LATEST_STABLE="$IMAGES_LATEST_STABLE $CI_REGISTRY_IMAGE/$arch:latest"; done' - - echo "$CI_REGISTRY_PASSWORD" | docker login $CI_REGISTRY --username $CI_REGISTRY_USER --password-stdin + - mkdir -p ~/.docker + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > ~/.docker/config.json docker-combine:latest: extends: .docker-combine From 1a50db36d3f40d25525128a45adc23038525d882 Mon Sep 17 00:00:00 2001 From: tusooa Date: Wed, 26 Apr 2023 07:20:35 -0400 Subject: [PATCH 32/89] Skip changelog entry for 3877 --- changelog.d/3877.skip | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 changelog.d/3877.skip diff --git a/changelog.d/3877.skip b/changelog.d/3877.skip new file mode 100644 index 000000000..e69de29bb From d5e83459460c4bb5202db8207d590f631285bdee Mon Sep 17 00:00:00 2001 From: tusooa Date: Wed, 26 Apr 2023 09:14:49 -0400 Subject: [PATCH 33/89] Do not use needs: in pipeline yaml --- .gitlab-ci.yml | 20 -------------------- changelog.d/3878.skip | 0 2 files changed, 20 deletions(-) create mode 100644 changelog.d/3878.skip diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 72ac4981d..7e2d487de 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -381,14 +381,6 @@ arm64-musl: entrypoint: [""] cache: {} dependencies: [] - needs: - - spec-build - - unit-testing - - unit-testing-erratic - - unit-testing-rum - - lint - - analysis - - cycles before_script: &before-kaniko - export CI_JOB_TIMESTAMP=$(date --utc -Iseconds) - export CI_VCS_REF=$CI_COMMIT_SHORT_SHA @@ -517,10 +509,6 @@ docker-combine:latest: extends: .docker-combine only: - develop@pleroma/pleroma - needs: - - 'kaniko-latest:linux/amd64' - - 'kaniko-latest:linux/arm64' - - 'kaniko-latest:linux/arm' script: - 'docker manifest create $IMAGE_TAG $IMAGES' - 'docker manifest push $IMAGE_TAG' @@ -533,10 +521,6 @@ docker-combine:stable: extends: .docker-combine only: - stable@pleroma/pleroma - needs: - - 'kaniko-stable:linux/amd64' - - 'kaniko-stable:linux/arm64' - - 'kaniko-stable:linux/arm' script: - 'docker manifest create $IMAGE_TAG $IMAGES' - 'docker manifest push $IMAGE_TAG' @@ -549,10 +533,6 @@ docker-combine:release: extends: .docker-combine only: - /^release/.*$/@pleroma/pleroma - needs: - - 'kaniko-release:linux/amd64' - - 'kaniko-release:linux/arm64' - - 'kaniko-release:linux/arm' script: - 'docker manifest create $IMAGE_TAG $IMAGES' - 'docker manifest push $IMAGE_TAG' diff --git a/changelog.d/3878.skip b/changelog.d/3878.skip new file mode 100644 index 000000000..e69de29bb From 85bdbb102e8fe4dc4a0eb765d8a2a01308161772 Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 4 Mar 2023 19:50:47 -0500 Subject: [PATCH 34/89] Add extraction process for oauth scopes --- lib/pleroma/web/api_spec/scopes/compiler.ex | 82 +++++++++++++++++++ lib/pleroma/web/api_spec/scopes/translator.ex | 10 +++ .../web/api_spec/scopes/compiler_test.exs | 56 +++++++++++++ 3 files changed, 148 insertions(+) create mode 100644 lib/pleroma/web/api_spec/scopes/compiler.ex create mode 100644 lib/pleroma/web/api_spec/scopes/translator.ex create mode 100644 test/pleroma/web/api_spec/scopes/compiler_test.exs diff --git a/lib/pleroma/web/api_spec/scopes/compiler.ex b/lib/pleroma/web/api_spec/scopes/compiler.ex new file mode 100644 index 000000000..162edc9a3 --- /dev/null +++ b/lib/pleroma/web/api_spec/scopes/compiler.ex @@ -0,0 +1,82 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2023 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Scopes.Compiler do + defmacro __before_compile__(_env) do + strings = __MODULE__.extract_all_scopes() + + quote do + def placeholder do + unquote do + Enum.map( + strings, + fn string -> + quote do + Pleroma.Web.Gettext.dgettext_noop( + "oauth_scopes", + unquote(string) + ) + end + end + ) + end + end + end + end + + def extract_all_scopes do + extract_all_scopes_from(Pleroma.Web.ApiSpec.spec()) + end + + def extract_all_scopes_from(specs) do + specs.paths + |> Enum.reduce([], fn + {_path, %{} = path_item}, acc -> + extract_routes(path_item) + |> Enum.flat_map(fn operation -> process_operation(operation) end) + |> Kernel.++(acc) + + {_, _}, acc -> + acc + end) + |> Enum.uniq() + end + + defp extract_routes(path_item) do + path_item + |> Map.from_struct() + |> Enum.map(fn {_method, path_item} -> path_item end) + |> Enum.filter(fn + %OpenApiSpex.Operation{} = _operation -> true + _ -> false + end) + end + + defp process_operation(operation) do + operation.security + |> Kernel.||([]) + |> Enum.flat_map(fn + %{"oAuth" => scopes} -> process_scopes(scopes) + _ -> [] + end) + end + + defp process_scopes(scopes) do + scopes + |> Enum.flat_map(fn scope -> + process_scope(scope) + end) + end + + def process_scope(scope) do + hierarchy = String.split(scope, ":") + + {_, list} = + Enum.reduce(hierarchy, {"", []}, fn comp, {cur, list} -> + {cur <> comp <> ":", [cur <> comp | list]} + end) + + list + end +end diff --git a/lib/pleroma/web/api_spec/scopes/translator.ex b/lib/pleroma/web/api_spec/scopes/translator.ex new file mode 100644 index 000000000..54eea3593 --- /dev/null +++ b/lib/pleroma/web/api_spec/scopes/translator.ex @@ -0,0 +1,10 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2023 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Scopes.Translator do + require Pleroma.Web.ApiSpec.Scopes.Compiler + require Pleroma.Web.Gettext + + @before_compile Pleroma.Web.ApiSpec.Scopes.Compiler +end diff --git a/test/pleroma/web/api_spec/scopes/compiler_test.exs b/test/pleroma/web/api_spec/scopes/compiler_test.exs new file mode 100644 index 000000000..99e1d343a --- /dev/null +++ b/test/pleroma/web/api_spec/scopes/compiler_test.exs @@ -0,0 +1,56 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2023 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Scopes.CompilerTest do + use ExUnit.Case, async: true + + alias Pleroma.Web.ApiSpec.Scopes.Compiler + + @dummy_response %{} + + @data %{ + paths: %{ + "/mew" => %OpenApiSpex.PathItem{ + post: %OpenApiSpex.Operation{ + security: [%{"oAuth" => ["a:b:c"]}], + responses: @dummy_response + }, + get: %OpenApiSpex.Operation{security: nil, responses: @dummy_response} + }, + "/mew2" => %OpenApiSpex.PathItem{ + post: %OpenApiSpex.Operation{ + security: [%{"oAuth" => ["d:e", "f:g"]}], + responses: @dummy_response + }, + get: %OpenApiSpex.Operation{security: nil, responses: @dummy_response} + } + } + } + + describe "process_scope/1" do + test "gives all higher-level scopes" do + scopes = Compiler.process_scope("admin:read:accounts") + + assert [_, _, _] = scopes + assert "admin" in scopes + assert "admin:read" in scopes + assert "admin:read:accounts" in scopes + end + end + + describe "extract_all_scopes_from/1" do + test "extracts scopes" do + scopes = Compiler.extract_all_scopes_from(@data) + + assert [_, _, _, _, _, _, _] = scopes + assert "a" in scopes + assert "a:b" in scopes + assert "a:b:c" in scopes + assert "d" in scopes + assert "d:e" in scopes + assert "f" in scopes + assert "f:g" in scopes + end + end +end From 530284e1b96b6daeb21995079ff21544e545aa7f Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 4 Mar 2023 19:57:42 -0500 Subject: [PATCH 35/89] Add extracted pot --- priv/gettext/oauth_scopes.pot | 261 ++++++++++++++++++++++++++++++++++ 1 file changed, 261 insertions(+) create mode 100644 priv/gettext/oauth_scopes.pot diff --git a/priv/gettext/oauth_scopes.pot b/priv/gettext/oauth_scopes.pot new file mode 100644 index 000000000..5f7b425f3 --- /dev/null +++ b/priv/gettext/oauth_scopes.pot @@ -0,0 +1,261 @@ +## This file is a PO Template file. +## +## "msgid"s here are often extracted from source code. +## Add new translations manually only if they're dynamic +## translations that can't be statically extracted. +## +## Run "mix gettext.extract" to bring this file up to +## date. Leave "msgstr"s empty as changing them here has no +## effect: edit them in PO (.po) files instead. +msgid "" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "follow" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "push" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:accounts" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:backups" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:blocks" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:bookmarks" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:chats" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:favourites" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:filters" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:follows" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:lists" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:notifications" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:reports" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:search" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:statuses" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:accounts" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:blocks" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:bookmarks" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:chats" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:conversations" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:favourites" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:filters" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:follow" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:follows" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:lists" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:media" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:mutes" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:notifications" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:reports" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:statuses" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:accounts" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:chats" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:invites" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:media_proxy_caches" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:reports" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:statuses" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:accounts" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:chats" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:follows" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:invites" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:media_proxy_caches" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:reports" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:statuses" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:media" +msgstr "" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:mutes" +msgstr "" From 6d0ebccdb03f6d19bccb01a6c3ae4e7f647aa678 Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 4 Mar 2023 19:58:05 -0500 Subject: [PATCH 36/89] Make webui use translated scope descriptions --- lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex index 73115e92a..7585c4d3e 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex @@ -8,7 +8,7 @@ <%= checkbox @form, :"scope_#{scope}", value: scope in @scopes && scope, checked_value: scope, unchecked_value: "", name: "authorization[scope][]" %> <%= label @form, :"scope_#{scope}", String.capitalize(scope) %> <%= if scope in @scopes && scope do %> - <%= String.capitalize(scope) %> + <%= scope %> <%= :"Elixir.Gettext".dgettext(Gettext, "oauth_scopes", scope) %> <% end %> <% else %> From b6dd194000a4a9ce23dc844758fa797f08a01550 Mon Sep 17 00:00:00 2001 From: tusooa Date: Tue, 2 May 2023 16:33:53 -0400 Subject: [PATCH 37/89] Add changelog --- changelog.d/3848.add | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3848.add diff --git a/changelog.d/3848.add b/changelog.d/3848.add new file mode 100644 index 000000000..d7b1b0a84 --- /dev/null +++ b/changelog.d/3848.add @@ -0,0 +1 @@ +Add OAuth scope descriptions From 9283c784a38194a937dc1a46a6b46478d159e434 Mon Sep 17 00:00:00 2001 From: tusooa Date: Tue, 2 May 2023 17:03:09 -0400 Subject: [PATCH 38/89] Add English translation for oauth scopes --- priv/gettext/en/LC_MESSAGES/oauth_scopes.po | 264 ++++++++++++++++++++ 1 file changed, 264 insertions(+) create mode 100644 priv/gettext/en/LC_MESSAGES/oauth_scopes.po diff --git a/priv/gettext/en/LC_MESSAGES/oauth_scopes.po b/priv/gettext/en/LC_MESSAGES/oauth_scopes.po new file mode 100644 index 000000000..105ca022b --- /dev/null +++ b/priv/gettext/en/LC_MESSAGES/oauth_scopes.po @@ -0,0 +1,264 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR Free Software Foundation, Inc. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"PO-Revision-Date: 2023-05-02 17:02-0400\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=CHARSET\n" +"Content-Transfer-Encoding: 8bit\n" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin" +msgstr "All admin access" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read" +msgstr "Read all using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write" +msgstr "Write all using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "follow" +msgstr "Read and write user relationships" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "push" +msgstr "Push notifications" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read" +msgstr "Read everything" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:accounts" +msgstr "Read information of all accounts" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:backups" +msgstr "Read your backups" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:blocks" +msgstr "Read block relationships" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:bookmarks" +msgstr "Read your bookmarks" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:chats" +msgstr "Read your chats" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:favourites" +msgstr "Read your favourites" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:filters" +msgstr "Read your filtering settings" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:follows" +msgstr "Read follow relationships" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:lists" +msgstr "Read your lists" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:notifications" +msgstr "Read your notifications" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:reports" +msgstr "Read your reports" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:search" +msgstr "Perform searches" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:statuses" +msgstr "Read all statuses you can see" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write" +msgstr "Write everything" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:accounts" +msgstr "Change your account information" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:blocks" +msgstr "Block or unblock someone" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:bookmarks" +msgstr "Add to or remove from your bookmarks" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:chats" +msgstr "Create or delete chats or chat messages, or mark them as read" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:conversations" +msgstr "Change recipients of, mark as read, or delete conversations" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:favourites" +msgstr "Favourite or unfavourite statuses" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:filters" +msgstr "Change your filtering settings" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:follow" +msgstr "Follow or unfollow someone" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:follows" +msgstr "Follow or unfollow someone" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:lists" +msgstr "Create, change or delete your lists" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:media" +msgstr "Upload media files or modify those you uploaded" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:mutes" +msgstr "Mute or unmute someone" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:notifications" +msgstr "Mark notifications as read" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:reports" +msgstr "Submit reports" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "write:statuses" +msgstr "Post, edit, reblog or react to statuses" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:accounts" +msgstr "Read all accounts using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:chats" +msgstr "Read all chats using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:invites" +msgstr "Read all invites using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:media_proxy_caches" +msgstr "Read media proxy caches using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:reports" +msgstr "Read all reports using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:read:statuses" +msgstr "Read all statuses using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:accounts" +msgstr "Change all accounts using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:chats" +msgstr "Change all chats using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:follows" +msgstr "Change follow relationships using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:invites" +msgstr "Invite or revoke an invite using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:media_proxy_caches" +msgstr "Change media proxy caches using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:reports" +msgstr "Handle reports using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "admin:write:statuses" +msgstr "Delete, change scope of, or mark as sensitive statuses using admin API" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:media" +msgstr "Read media attachments" + +#, elixir-autogen, elixir-format +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +msgid "read:mutes" +msgstr "Read mute relationships" From 99f157e2804327d4ad3f47b83638060f774a582b Mon Sep 17 00:00:00 2001 From: tusooa Date: Tue, 2 May 2023 22:12:15 -0400 Subject: [PATCH 39/89] Fix MR pipelines not having build and test jobs --- .gitlab-ci.yml | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2eee3bc68..b70415ac6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -56,24 +56,27 @@ check-changelog: - apk add git - sh ./tools/check-changelog +.build_changes_policy: + rules: + - changes: + - ".gitlab-ci.yml" + - "**/*.ex" + - "**/*.exs" + - "mix.lock" + build: + extends: .build_changes_policy stage: build - only: - changes: &build_changes_policy - - ".gitlab-ci.yml" - - "**/*.ex" - - "**/*.exs" - - "mix.lock" script: - mix compile --force spec-build: stage: test - only: - changes: - - ".gitlab-ci.yml" - - "lib/pleroma/web/api_spec/**/*.ex" - - "lib/pleroma/web/api_spec.ex" + rules: + - changes: + - ".gitlab-ci.yml" + - "lib/pleroma/web/api_spec/**/*.ex" + - "lib/pleroma/web/api_spec.ex" artifacts: paths: - spec.json @@ -95,9 +98,8 @@ benchmark: - mix pleroma.load_testing unit-testing: + extends: .build_changes_policy stage: test - only: - changes: *build_changes_policy cache: &testing_cache_policy <<: *global_cache_policy policy: pull @@ -118,11 +120,10 @@ unit-testing: path: coverage.xml unit-testing-erratic: + extends: .build_changes_policy stage: test retry: 2 allow_failure: true - only: - changes: *build_changes_policy cache: &testing_cache_policy <<: *global_cache_policy policy: pull @@ -153,9 +154,8 @@ unit-testing-erratic: # - mix test --trace --only federated unit-testing-rum: + extends: .build_changes_policy stage: test - only: - changes: *build_changes_policy cache: *testing_cache_policy services: - name: minibikini/postgres-with-rum:12 @@ -171,10 +171,9 @@ unit-testing-rum: - mix test --preload-modules lint: + extends: .build_changes_policy image: ¤t_elixir elixir:1.12-alpine stage: test - only: - changes: *build_changes_policy cache: *testing_cache_policy before_script: ¤t_bfr_script - apk update @@ -186,18 +185,16 @@ lint: - mix format --check-formatted analysis: + extends: .build_changes_policy stage: test - only: - changes: *build_changes_policy cache: *testing_cache_policy script: - mix credo --strict --only=warnings,todo,fixme,consistency,readability cycles: + extends: .build_changes_policy image: *current_elixir stage: test - only: - changes: *build_changes_policy cache: {} before_script: *current_bfr_script script: From 6a3fd8e01404f7fc4415626300d81c65c045a7d6 Mon Sep 17 00:00:00 2001 From: tusooa Date: Tue, 2 May 2023 22:16:00 -0400 Subject: [PATCH 40/89] Do not count for renames when diffing --- tools/check-changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/check-changelog b/tools/check-changelog index 1409be9f0..60692033f 100644 --- a/tools/check-changelog +++ b/tools/check-changelog @@ -5,7 +5,7 @@ echo "looking for change log" git remote add upstream https://git.pleroma.social/pleroma/pleroma.git git fetch upstream ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}:refs/remotes/upstream/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME -git diff --raw upstream/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME HEAD -- changelog.d | \ +git diff --raw --no-renames upstream/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME HEAD -- changelog.d | \ grep ' A\t' | grep '\.\(skip\|add\|remove\|fix\|security\)$' ret=$? From 4fd96b24aef831095ac8b3494fe4c59d8c9de333 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 5 May 2023 10:53:09 +0200 Subject: [PATCH 41/89] AddRemoveValidator: Use User.fetch_by_ap_id instead of upgrade_user_from_ap_id --- .../web/activity_pub/object_validators/add_remove_validator.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex b/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex index 5202db7f1..db3259550 100644 --- a/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex @@ -73,6 +73,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do end defp maybe_refetch_user(%User{ap_id: ap_id}) do - Pleroma.Web.ActivityPub.Transmogrifier.upgrade_user_from_ap_id(ap_id) + # Maybe it could use User.get_or_fetch_by_ap_id to avoid refreshing too often + User.fetch_by_ap_id(ap_id) end end From 0903c416456d137fa40bef231c31f038c8c20e9f Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 5 May 2023 10:54:16 +0200 Subject: [PATCH 42/89] User: Stop relying on ap_enabled --- lib/pleroma/user.ex | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index f6e30555c..2a0faed77 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1061,11 +1061,7 @@ defmodule Pleroma.User do end def maybe_direct_follow(%User{} = follower, %User{} = followed) do - if not ap_enabled?(followed) do - follow(follower, followed) - else - {:ok, follower, followed} - end + {:ok, follower, followed} end @doc "A mass follow for local users. Respects blocks in both directions but does not create activities." From 606f78f5e5781a59cb9d083e8085144cca326cc6 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 5 May 2023 10:55:08 +0200 Subject: [PATCH 43/89] ActivityPub: Stop relying on ap_enabled and upgrade_user_from_ap_id --- lib/pleroma/web/activity_pub/activity_pub.ex | 28 +++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index f22756015..2fae98fad 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1751,24 +1751,20 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def make_user_from_ap_id(ap_id, additional \\ []) do user = User.get_cached_by_ap_id(ap_id) - if user && !User.ap_enabled?(user) do - Transmogrifier.upgrade_user_from_ap_id(ap_id) - else - with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, additional) do - {:ok, _pid} = Task.start(fn -> pinned_fetch_task(data) end) + with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, additional) do + {:ok, _pid} = Task.start(fn -> pinned_fetch_task(data) end) - if user do - user - |> User.remote_user_changeset(data) - |> User.update_and_set_cache() - else - maybe_handle_clashing_nickname(data) + if user do + user + |> User.remote_user_changeset(data) + |> User.update_and_set_cache() + else + maybe_handle_clashing_nickname(data) - data - |> User.remote_user_changeset() - |> Repo.insert() - |> User.set_cache() - end + data + |> User.remote_user_changeset() + |> Repo.insert() + |> User.set_cache() end end end From 3962253cf16d19237b00b5e01573aed10fb596bd Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 5 May 2023 10:55:46 +0200 Subject: [PATCH 44/89] Publisher: Stop filtering via ap_enabled?/1 --- lib/pleroma/web/activity_pub/publisher.ex | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 6c1ba76a3..af6aa0781 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -199,7 +199,6 @@ defmodule Pleroma.Web.ActivityPub.Publisher do inboxes = recipients - |> Enum.filter(&User.ap_enabled?/1) |> Enum.map(fn actor -> actor.inbox end) |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) |> Instances.filter_reachable() @@ -241,7 +240,6 @@ defmodule Pleroma.Web.ActivityPub.Publisher do json = Jason.encode!(data) recipients(actor, activity) - |> Enum.filter(fn user -> User.ap_enabled?(user) end) |> Enum.map(fn %User{} = user -> determine_inbox(activity, user) end) From 2ee483ba41eb4f01c3ca763c30dcaa07d3b258f9 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 5 May 2023 11:00:40 +0200 Subject: [PATCH 45/89] Transmogrifier: Remove upgrade_user_from_ap_id --- .../web/activity_pub/transmogrifier.ex | 20 ------ .../web/activity_pub/transmogrifier_test.exs | 64 ------------------- 2 files changed, 84 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 3141f8437..ea7a6a810 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -20,7 +20,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.Federator - alias Pleroma.Workers.TransmogrifierWorker import Ecto.Query @@ -968,25 +967,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Repo.update_all([]) end - def upgrade_user_from_ap_id(ap_id) do - with %User{local: false} = user <- User.get_cached_by_ap_id(ap_id), - {:ok, data} <- ActivityPub.fetch_and_prepare_user_from_ap_id(ap_id), - {:ok, user} <- update_user(user, data) do - {:ok, _pid} = Task.start(fn -> ActivityPub.pinned_fetch_task(user) end) - TransmogrifierWorker.enqueue("user_upgrade", %{"user_id" => user.id}) - {:ok, user} - else - %User{} = user -> {:ok, user} - e -> e - end - end - - defp update_user(user, data) do - user - |> User.remote_user_changeset(data) - |> User.update_and_set_cache() - end - def maybe_fix_user_url(%{"url" => url} = data) when is_map(url) do Map.put(data, "url", url["href"]) end diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index 6b4636d22..588f46296 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -8,7 +8,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do alias Pleroma.Activity alias Pleroma.Object - alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils @@ -339,69 +338,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do end end - describe "user upgrade" do - test "it upgrades a user to activitypub" do - user = - insert(:user, %{ - nickname: "rye@niu.moe", - local: false, - ap_id: "https://niu.moe/users/rye", - follower_address: User.ap_followers(%User{nickname: "rye@niu.moe"}) - }) - - user_two = insert(:user) - Pleroma.FollowingRelationship.follow(user_two, user, :follow_accept) - - {:ok, activity} = CommonAPI.post(user, %{status: "test"}) - {:ok, unrelated_activity} = CommonAPI.post(user_two, %{status: "test"}) - assert "http://localhost:4001/users/rye@niu.moe/followers" in activity.recipients - - user = User.get_cached_by_id(user.id) - assert user.note_count == 1 - - {:ok, user} = Transmogrifier.upgrade_user_from_ap_id("https://niu.moe/users/rye") - ObanHelpers.perform_all() - - assert user.ap_enabled - assert user.note_count == 1 - assert user.follower_address == "https://niu.moe/users/rye/followers" - assert user.following_address == "https://niu.moe/users/rye/following" - - user = User.get_cached_by_id(user.id) - assert user.note_count == 1 - - activity = Activity.get_by_id(activity.id) - assert user.follower_address in activity.recipients - - assert %{ - "url" => [ - %{ - "href" => - "https://cdn.niu.moe/accounts/avatars/000/033/323/original/fd7f8ae0b3ffedc9.jpeg" - } - ] - } = user.avatar - - assert %{ - "url" => [ - %{ - "href" => - "https://cdn.niu.moe/accounts/headers/000/033/323/original/850b3448fa5fd477.png" - } - ] - } = user.banner - - refute "..." in activity.recipients - - unrelated_activity = Activity.get_by_id(unrelated_activity.id) - refute user.follower_address in unrelated_activity.recipients - - user_two = User.get_cached_by_id(user_two.id) - assert User.following?(user_two, user) - refute "..." in User.following(user_two) - end - end - describe "actor rewriting" do test "it fixes the actor URL property to be a proper URI" do data = %{ From e17265a7a248db7514b091d47ae79d1a2de5b440 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 5 May 2023 11:01:29 +0200 Subject: [PATCH 46/89] TransmogrifierWorker: Remove obsolete worker --- .../web/activity_pub/transmogrifier.ex | 22 ------------------- lib/pleroma/workers/transmogrifier_worker.ex | 18 --------------- 2 files changed, 40 deletions(-) delete mode 100644 lib/pleroma/workers/transmogrifier_worker.ex diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index ea7a6a810..0e6c429f9 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -945,28 +945,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do defp strip_internal_tags(object), do: object - def perform(:user_upgrade, user) do - # we pass a fake user so that the followers collection is stripped away - old_follower_address = User.ap_followers(%User{nickname: user.nickname}) - - from( - a in Activity, - where: ^old_follower_address in a.recipients, - update: [ - set: [ - recipients: - fragment( - "array_replace(?,?,?)", - a.recipients, - ^old_follower_address, - ^user.follower_address - ) - ] - ] - ) - |> Repo.update_all([]) - end - def maybe_fix_user_url(%{"url" => url} = data) when is_map(url) do Map.put(data, "url", url["href"]) end diff --git a/lib/pleroma/workers/transmogrifier_worker.ex b/lib/pleroma/workers/transmogrifier_worker.ex deleted file mode 100644 index 1f3f5385e..000000000 --- a/lib/pleroma/workers/transmogrifier_worker.ex +++ /dev/null @@ -1,18 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.TransmogrifierWorker do - alias Pleroma.User - - use Pleroma.Workers.WorkerHelper, queue: "transmogrifier" - - @impl Oban.Worker - def perform(%Job{args: %{"op" => "user_upgrade", "user_id" => user_id}}) do - user = User.get_cached_by_id(user_id) - Pleroma.Web.ActivityPub.Transmogrifier.perform(:user_upgrade, user) - end - - @impl Oban.Worker - def timeout(_job), do: :timer.seconds(5) -end From 8181be89a28ec293cb36fccc2324bc78cc118d3a Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 5 May 2023 11:02:58 +0200 Subject: [PATCH 47/89] Federator: Stop using ap_enabled?/1 --- lib/pleroma/web/federator.ex | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 318b6cb11..84b77cda1 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -6,7 +6,6 @@ defmodule Pleroma.Web.Federator do alias Pleroma.Activity alias Pleroma.Object.Containment alias Pleroma.User - alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Federator.Publisher @@ -80,7 +79,7 @@ defmodule Pleroma.Web.Federator do # NOTE: we use the actor ID to do the containment, this is fine because an # actor shouldn't be acting on objects outside their own AP server. - with {_, {:ok, _user}} <- {:actor, ap_enabled_actor(actor)}, + with {_, {:ok, _user}} <- {:actor, User.get_or_fetch_by_ap_id(actor)}, nil <- Activity.normalize(params["id"]), {_, :ok} <- {:correct_origin?, Containment.contain_origin_from_id(actor, params)}, @@ -110,14 +109,4 @@ defmodule Pleroma.Web.Federator do {:error, e} end end - - def ap_enabled_actor(id) do - user = User.get_cached_by_ap_id(id) - - if User.ap_enabled?(user) do - {:ok, user} - else - ActivityPub.make_user_from_ap_id(id) - end - end end From 9dfa1c4be0d9c37222a299c212607407f93d6596 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 5 May 2023 11:07:18 +0200 Subject: [PATCH 48/89] ActivityPub: Mark fetch_and_prepare_user_from_ap_id/1 as private --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 2fae98fad..9175e9e13 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1668,7 +1668,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end - def fetch_and_prepare_user_from_ap_id(ap_id, additional \\ []) do + defp fetch_and_prepare_user_from_ap_id(ap_id, additional) do with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id), {:ok, data} <- user_data_from_user_object(data, additional) do {:ok, maybe_update_follow_information(data)} From 238edc30dec2cb48b3a6eca48397d5202060726e Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 5 May 2023 11:08:26 +0200 Subject: [PATCH 49/89] User: Remove ap_enabled?/1 --- lib/pleroma/user.ex | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 2a0faed77..eff82afd9 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2147,10 +2147,6 @@ defmodule Pleroma.User do end end - def ap_enabled?(%User{local: true}), do: true - def ap_enabled?(%User{ap_enabled: ap_enabled}), do: ap_enabled - def ap_enabled?(_), do: false - @doc "Gets or fetch a user by uri or nickname." @spec get_or_fetch(String.t()) :: {:ok, User.t()} | {:error, String.t()} def get_or_fetch("http://" <> _host = uri), do: get_or_fetch_by_ap_id(uri) From fcd49e3985d34bd3b783791e12e638f66cab0a9b Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 5 May 2023 11:09:15 +0200 Subject: [PATCH 50/89] User: Remove ap_enabled field --- lib/pleroma/user.ex | 3 --- lib/pleroma/web/activity_pub/activity_pub.ex | 1 - .../20230504173400_remove_user_ap_enabled.exs | 13 +++++++++++++ test/pleroma/user_test.exs | 11 +++-------- .../activity_pub/activity_pub_controller_test.exs | 1 - .../web/activity_pub/activity_pub_test.exs | 1 - test/pleroma/web/activity_pub/publisher_test.exs | 15 +++++---------- test/pleroma/web/common_api_test.exs | 2 +- test/pleroma/web/federator_test.exs | 6 ++---- test/support/factory.ex | 1 - 10 files changed, 24 insertions(+), 30 deletions(-) create mode 100644 priv/repo/migrations/20230504173400_remove_user_ap_enabled.exs diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index eff82afd9..ce125d608 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -124,7 +124,6 @@ defmodule Pleroma.User do field(:domain_blocks, {:array, :string}, default: []) field(:is_active, :boolean, default: true) field(:no_rich_text, :boolean, default: false) - field(:ap_enabled, :boolean, default: false) field(:is_moderator, :boolean, default: false) field(:is_admin, :boolean, default: false) field(:show_role, :boolean, default: true) @@ -488,7 +487,6 @@ defmodule Pleroma.User do :nickname, :public_key, :avatar, - :ap_enabled, :banner, :is_locked, :last_refreshed_at, @@ -1894,7 +1892,6 @@ defmodule Pleroma.User do confirmation_token: nil, domain_blocks: [], is_active: false, - ap_enabled: false, is_moderator: false, is_admin: false, mascot: nil, diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 9175e9e13..1a10aef1d 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1547,7 +1547,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do %{ ap_id: data["id"], uri: get_actor_url(data["url"]), - ap_enabled: true, banner: normalize_image(data["image"]), fields: fields, emoji: emojis, diff --git a/priv/repo/migrations/20230504173400_remove_user_ap_enabled.exs b/priv/repo/migrations/20230504173400_remove_user_ap_enabled.exs new file mode 100644 index 000000000..0aea41324 --- /dev/null +++ b/priv/repo/migrations/20230504173400_remove_user_ap_enabled.exs @@ -0,0 +1,13 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2023 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.RemoveUserApEnabled do + use Ecto.Migration + + def change do + alter table(:users) do + remove(:ap_enabled, :boolean, default: false, null: false) + end + end +end diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index c16312a65..7f60b959a 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -1844,7 +1844,6 @@ defmodule Pleroma.UserTest do confirmation_token: "qqqq", domain_blocks: ["lain.com"], is_active: false, - ap_enabled: true, is_moderator: true, is_admin: true, mascot: %{"a" => "b"}, @@ -1885,7 +1884,6 @@ defmodule Pleroma.UserTest do confirmation_token: nil, domain_blocks: [], is_active: false, - ap_enabled: false, is_moderator: false, is_admin: false, mascot: nil, @@ -2473,8 +2471,7 @@ defmodule Pleroma.UserTest do insert(:user, local: false, follower_address: "http://localhost:4001/users/masto_closed/followers", - following_address: "http://localhost:4001/users/masto_closed/following", - ap_enabled: true + following_address: "http://localhost:4001/users/masto_closed/following" ) assert other_user.following_count == 0 @@ -2495,8 +2492,7 @@ defmodule Pleroma.UserTest do insert(:user, local: false, follower_address: "http://localhost:4001/users/masto_closed/followers", - following_address: "http://localhost:4001/users/masto_closed/following", - ap_enabled: true + following_address: "http://localhost:4001/users/masto_closed/following" ) assert other_user.following_count == 0 @@ -2517,8 +2513,7 @@ defmodule Pleroma.UserTest do insert(:user, local: false, follower_address: "http://localhost:4001/users/masto_closed/followers", - following_address: "http://localhost:4001/users/masto_closed/following", - ap_enabled: true + following_address: "http://localhost:4001/users/masto_closed/following" ) assert other_user.following_count == 0 diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index ef91066c1..62eb9b5a3 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -575,7 +575,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do user = insert(:user, ap_id: "https://mastodon.example.org/users/raymoo", - ap_enabled: true, local: false, last_refreshed_at: nil ) diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index a6f8b6152..54fc6ef0a 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -174,7 +174,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do {:ok, user} = ActivityPub.make_user_from_ap_id(user_id) assert user.ap_id == user_id assert user.nickname == "admin@mastodon.example.org" - assert user.ap_enabled assert user.follower_address == "http://mastodon.example.org/users/admin/followers" end diff --git a/test/pleroma/web/activity_pub/publisher_test.exs b/test/pleroma/web/activity_pub/publisher_test.exs index e2db3d575..c5137cbb7 100644 --- a/test/pleroma/web/activity_pub/publisher_test.exs +++ b/test/pleroma/web/activity_pub/publisher_test.exs @@ -276,8 +276,7 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do follower = insert(:user, %{ local: false, - inbox: "https://domain.com/users/nick1/inbox", - ap_enabled: true + inbox: "https://domain.com/users/nick1/inbox" }) actor = insert(:user, follower_address: follower.ap_id) @@ -313,8 +312,7 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do follower = insert(:user, %{ local: false, - inbox: "https://domain.com/users/nick1/inbox", - ap_enabled: true + inbox: "https://domain.com/users/nick1/inbox" }) actor = insert(:user, follower_address: follower.ap_id) @@ -348,8 +346,7 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do follower = insert(:user, %{ local: false, - inbox: "https://domain.com/users/nick1/inbox", - ap_enabled: true + inbox: "https://domain.com/users/nick1/inbox" }) actor = insert(:user, follower_address: follower.ap_id) @@ -382,15 +379,13 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do fetcher = insert(:user, local: false, - inbox: "https://domain.com/users/nick1/inbox", - ap_enabled: true + inbox: "https://domain.com/users/nick1/inbox" ) another_fetcher = insert(:user, local: false, - inbox: "https://domain2.com/users/nick1/inbox", - ap_enabled: true + inbox: "https://domain2.com/users/nick1/inbox" ) actor = insert(:user) diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 5c9103e9f..5bd0df0b2 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -1328,7 +1328,7 @@ defmodule Pleroma.Web.CommonAPITest do test "cancels a pending follow for a remote user" do follower = insert(:user) - followed = insert(:user, is_locked: true, local: false, ap_enabled: true) + followed = insert(:user, is_locked: true, local: false) assert {:ok, follower, followed, %{id: activity_id, data: %{"state" => "pending"}}} = CommonAPI.follow(follower, followed) diff --git a/test/pleroma/web/federator_test.exs b/test/pleroma/web/federator_test.exs index 1ffe6aae1..6826e6c2f 100644 --- a/test/pleroma/web/federator_test.exs +++ b/test/pleroma/web/federator_test.exs @@ -78,16 +78,14 @@ defmodule Pleroma.Web.FederatorTest do local: false, nickname: "nick1@domain.com", ap_id: "https://domain.com/users/nick1", - inbox: inbox1, - ap_enabled: true + inbox: inbox1 }) insert(:user, %{ local: false, nickname: "nick2@domain2.com", ap_id: "https://domain2.com/users/nick2", - inbox: inbox2, - ap_enabled: true + inbox: inbox2 }) dt = NaiveDateTime.utc_now() diff --git a/test/support/factory.ex b/test/support/factory.ex index 09f02458c..d94544717 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -50,7 +50,6 @@ defmodule Pleroma.Factory do last_refreshed_at: NaiveDateTime.utc_now(), notification_settings: %Pleroma.User.NotificationSetting{}, multi_factor_authentication_settings: %Pleroma.MFA.Settings{}, - ap_enabled: true, keys: pem } From c63bf6a040ffd982da43e980209a7d96e87e4d7f Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 5 May 2023 11:13:50 +0200 Subject: [PATCH 51/89] Add changelog for !3880 --- changelog.d/3880.remove | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3880.remove diff --git a/changelog.d/3880.remove b/changelog.d/3880.remove new file mode 100644 index 000000000..113c76c85 --- /dev/null +++ b/changelog.d/3880.remove @@ -0,0 +1 @@ +Cleanup OStatus-era user upgrades and ap_enabled indicator \ No newline at end of file From 163e82bab1cc12ad323bfc9c1fcada11fa3b223a Mon Sep 17 00:00:00 2001 From: tusooa Date: Tue, 9 May 2023 19:25:39 -0400 Subject: [PATCH 52/89] Allow lang attribute --- changelog.d/3882.add | 1 + priv/scrubbers/default.ex | 77 ++++++++++++++-------------- test/pleroma/web/common_api_test.exs | 11 ++++ 3 files changed, 51 insertions(+), 38 deletions(-) create mode 100644 changelog.d/3882.add diff --git a/changelog.d/3882.add b/changelog.d/3882.add new file mode 100644 index 000000000..4712de1dc --- /dev/null +++ b/changelog.d/3882.add @@ -0,0 +1 @@ +Allow lang attribute in status text diff --git a/priv/scrubbers/default.ex b/priv/scrubbers/default.ex index e10e3ec87..d1215d2e0 100644 --- a/priv/scrubbers/default.ex +++ b/priv/scrubbers/default.ex @@ -33,35 +33,35 @@ defmodule Pleroma.HTML.Scrubber.Default do "ugc" ]) - Meta.allow_tag_with_these_attributes(:a, ["name", "title"]) + Meta.allow_tag_with_these_attributes(:a, ["name", "title", "lang"]) - Meta.allow_tag_with_these_attributes(:abbr, ["title"]) + Meta.allow_tag_with_these_attributes(:abbr, ["title", "lang"]) - Meta.allow_tag_with_these_attributes(:b, []) - Meta.allow_tag_with_these_attributes(:blockquote, []) - Meta.allow_tag_with_these_attributes(:br, []) - Meta.allow_tag_with_these_attributes(:code, []) - Meta.allow_tag_with_these_attributes(:del, []) - Meta.allow_tag_with_these_attributes(:em, []) - Meta.allow_tag_with_these_attributes(:hr, []) - Meta.allow_tag_with_these_attributes(:i, []) - Meta.allow_tag_with_these_attributes(:li, []) - Meta.allow_tag_with_these_attributes(:ol, []) - Meta.allow_tag_with_these_attributes(:p, []) - Meta.allow_tag_with_these_attributes(:pre, []) - Meta.allow_tag_with_these_attributes(:strong, []) - Meta.allow_tag_with_these_attributes(:sub, []) - Meta.allow_tag_with_these_attributes(:sup, []) - Meta.allow_tag_with_these_attributes(:ruby, []) - Meta.allow_tag_with_these_attributes(:rb, []) - Meta.allow_tag_with_these_attributes(:rp, []) - Meta.allow_tag_with_these_attributes(:rt, []) - Meta.allow_tag_with_these_attributes(:rtc, []) - Meta.allow_tag_with_these_attributes(:u, []) - Meta.allow_tag_with_these_attributes(:ul, []) + Meta.allow_tag_with_these_attributes(:b, ["lang"]) + Meta.allow_tag_with_these_attributes(:blockquote, ["lang"]) + Meta.allow_tag_with_these_attributes(:br, ["lang"]) + Meta.allow_tag_with_these_attributes(:code, ["lang"]) + Meta.allow_tag_with_these_attributes(:del, ["lang"]) + Meta.allow_tag_with_these_attributes(:em, ["lang"]) + Meta.allow_tag_with_these_attributes(:hr, ["lang"]) + Meta.allow_tag_with_these_attributes(:i, ["lang"]) + Meta.allow_tag_with_these_attributes(:li, ["lang"]) + Meta.allow_tag_with_these_attributes(:ol, ["lang"]) + Meta.allow_tag_with_these_attributes(:p, ["lang"]) + Meta.allow_tag_with_these_attributes(:pre, ["lang"]) + Meta.allow_tag_with_these_attributes(:strong, ["lang"]) + Meta.allow_tag_with_these_attributes(:sub, ["lang"]) + Meta.allow_tag_with_these_attributes(:sup, ["lang"]) + Meta.allow_tag_with_these_attributes(:ruby, ["lang"]) + Meta.allow_tag_with_these_attributes(:rb, ["lang"]) + Meta.allow_tag_with_these_attributes(:rp, ["lang"]) + Meta.allow_tag_with_these_attributes(:rt, ["lang"]) + Meta.allow_tag_with_these_attributes(:rtc, ["lang"]) + Meta.allow_tag_with_these_attributes(:u, ["lang"]) + Meta.allow_tag_with_these_attributes(:ul, ["lang"]) Meta.allow_tag_with_this_attribute_values(:span, "class", ["h-card", "recipients-inline"]) - Meta.allow_tag_with_these_attributes(:span, []) + Meta.allow_tag_with_these_attributes(:span, ["lang"]) Meta.allow_tag_with_this_attribute_values(:code, "class", ["inline"]) @@ -77,29 +77,30 @@ defmodule Pleroma.HTML.Scrubber.Default do "width", "height", "title", - "alt" + "alt", + "lang" ]) end if Pleroma.Config.get([:markup, :allow_tables]) do - Meta.allow_tag_with_these_attributes(:table, []) - Meta.allow_tag_with_these_attributes(:tbody, []) - Meta.allow_tag_with_these_attributes(:td, []) - Meta.allow_tag_with_these_attributes(:th, []) - Meta.allow_tag_with_these_attributes(:thead, []) - Meta.allow_tag_with_these_attributes(:tr, []) + Meta.allow_tag_with_these_attributes(:table, ["lang"]) + Meta.allow_tag_with_these_attributes(:tbody, ["lang"]) + Meta.allow_tag_with_these_attributes(:td, ["lang"]) + Meta.allow_tag_with_these_attributes(:th, ["lang"]) + Meta.allow_tag_with_these_attributes(:thead, ["lang"]) + Meta.allow_tag_with_these_attributes(:tr, ["lang"]) end if Pleroma.Config.get([:markup, :allow_headings]) do - Meta.allow_tag_with_these_attributes(:h1, []) - Meta.allow_tag_with_these_attributes(:h2, []) - Meta.allow_tag_with_these_attributes(:h3, []) - Meta.allow_tag_with_these_attributes(:h4, []) - Meta.allow_tag_with_these_attributes(:h5, []) + Meta.allow_tag_with_these_attributes(:h1, ["lang"]) + Meta.allow_tag_with_these_attributes(:h2, ["lang"]) + Meta.allow_tag_with_these_attributes(:h3, ["lang"]) + Meta.allow_tag_with_these_attributes(:h4, ["lang"]) + Meta.allow_tag_with_these_attributes(:h5, ["lang"]) end if Pleroma.Config.get([:markup, :allow_fonts]) do - Meta.allow_tag_with_these_attributes(:font, ["face"]) + Meta.allow_tag_with_these_attributes(:font, ["face", "lang"]) end Meta.strip_everything_not_covered() diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 5c9103e9f..968d826a2 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -527,6 +527,17 @@ defmodule Pleroma.Web.CommonAPITest do assert Object.tags(object) == ["ساٴين‌س"] end + test "allows lang attribute" do + user = insert(:user) + text = ~s{something

random

} + + {:ok, activity} = CommonAPI.post(user, %{status: text, content_type: "text/html"}) + + object = Object.normalize(activity, fetch: false) + + assert object.data["content"] == text + end + test "double dot in link is allowed" do user = insert(:user) text = "https://example.to/something..mp3" From be5c5118cb296792d82c007fe3b83d8795dca4b0 Mon Sep 17 00:00:00 2001 From: tusooa Date: Tue, 9 May 2023 21:04:01 -0400 Subject: [PATCH 53/89] Make sure object refetching follows update rules --- changelog.d/3883.fix | 1 + lib/pleroma/object/fetcher.ex | 86 ++++---------------- lib/pleroma/object/updater.ex | 49 +++++++++++ lib/pleroma/web/activity_pub/side_effects.ex | 36 ++------ test/pleroma/object/fetcher_test.exs | 84 ++++++++++++++++--- 5 files changed, 145 insertions(+), 111 deletions(-) create mode 100644 changelog.d/3883.fix diff --git a/changelog.d/3883.fix b/changelog.d/3883.fix new file mode 100644 index 000000000..6824f2013 --- /dev/null +++ b/changelog.d/3883.fix @@ -0,0 +1 @@ +Fix abnormal behaviour when refetching a poll diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index a9a9eeeed..cc3772563 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -8,77 +8,30 @@ defmodule Pleroma.Object.Fetcher do alias Pleroma.Maps alias Pleroma.Object alias Pleroma.Object.Containment - alias Pleroma.Repo alias Pleroma.Signature alias Pleroma.Web.ActivityPub.InternalFetchActor + alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.ObjectValidator + alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.Federator require Logger require Pleroma.Constants - defp touch_changeset(changeset) do - updated_at = - NaiveDateTime.utc_now() - |> NaiveDateTime.truncate(:second) - - Ecto.Changeset.put_change(changeset, :updated_at, updated_at) - end - - defp maybe_reinject_internal_fields(%{data: %{} = old_data}, new_data) do - has_history? = fn - %{"formerRepresentations" => %{"orderedItems" => list}} when is_list(list) -> true - _ -> false - end - - internal_fields = Map.take(old_data, Pleroma.Constants.object_internal_fields()) - - remote_history_exists? = has_history?.(new_data) - - # If the remote history exists, we treat that as the only source of truth. - new_data = - if has_history?.(old_data) and not remote_history_exists? do - Map.put(new_data, "formerRepresentations", old_data["formerRepresentations"]) - else - new_data - end - - # If the remote does not have history information, we need to manage it ourselves - new_data = - if not remote_history_exists? do - changed? = - Pleroma.Constants.status_updatable_fields() - |> Enum.any?(fn field -> Map.get(old_data, field) != Map.get(new_data, field) end) - - %{updated_object: updated_object} = - new_data - |> Object.Updater.maybe_update_history(old_data, - updated: changed?, - use_history_in_new_object?: false - ) - - updated_object - else - new_data - end - - Map.merge(new_data, internal_fields) - end - - defp maybe_reinject_internal_fields(_, new_data), do: new_data - @spec reinject_object(struct(), map()) :: {:ok, Object.t()} | {:error, any()} - defp reinject_object(%Object{data: %{"type" => "Question"}} = object, new_data) do + defp reinject_object(%Object{data: %{}} = object, new_data) do Logger.debug("Reinjecting object #{new_data["id"]}") - with data <- maybe_reinject_internal_fields(object, new_data), - {:ok, data, _} <- ObjectValidator.validate(data, %{}), - changeset <- Object.change(object, %{data: data}), - changeset <- touch_changeset(changeset), - {:ok, object} <- Repo.insert_or_update(changeset), - {:ok, object} <- Object.set_cache(object) do - {:ok, object} + with {:ok, new_data, _} <- ObjectValidator.validate(new_data, %{}), + {:ok, new_data} <- MRF.filter(new_data), + {:ok, new_object, _} <- + Object.Updater.do_update_and_invalidate_cache( + object, + new_data, + _touch_changeset? = true + ) do + {:ok, new_object} else e -> Logger.error("Error while processing object: #{inspect(e)}") @@ -86,20 +39,11 @@ defmodule Pleroma.Object.Fetcher do end end - defp reinject_object(%Object{} = object, new_data) do - Logger.debug("Reinjecting object #{new_data["id"]}") - - with new_data <- Transmogrifier.fix_object(new_data), - data <- maybe_reinject_internal_fields(object, new_data), - changeset <- Object.change(object, %{data: data}), - changeset <- touch_changeset(changeset), - {:ok, object} <- Repo.insert_or_update(changeset), - {:ok, object} <- Object.set_cache(object) do + defp reinject_object(_, new_data) do + with {:ok, object, _} <- Pipeline.common_pipeline(new_data, local: false) do {:ok, object} else - e -> - Logger.error("Error while processing object: #{inspect(e)}") - {:error, e} + e -> e end end diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex index ab38d3ed2..0c523ed35 100644 --- a/lib/pleroma/object/updater.ex +++ b/lib/pleroma/object/updater.ex @@ -5,6 +5,9 @@ defmodule Pleroma.Object.Updater do require Pleroma.Constants + alias Pleroma.Object + alias Pleroma.Repo + def update_content_fields(orig_object_data, updated_object) do Pleroma.Constants.status_updatable_fields() |> Enum.reduce( @@ -237,4 +240,50 @@ defmodule Pleroma.Object.Updater do {:history_items, e} -> e end end + + defp maybe_touch_changeset(changeset, true) do + updated_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.truncate(:second) + + Ecto.Changeset.put_change(changeset, :updated_at, updated_at) + end + + defp maybe_touch_changeset(changeset, _), do: changeset + + def do_update_and_invalidate_cache(orig_object, updated_object, touch_changeset? \\ false) do + orig_object_ap_id = updated_object["id"] + orig_object = Object.get_by_ap_id(orig_object_ap_id) + orig_object_data = orig_object.data + + %{ + updated_data: updated_object_data, + updated: updated, + used_history_in_new_object?: used_history_in_new_object? + } = make_new_object_data_from_update_object(orig_object_data, updated_object) + + changeset = + orig_object + |> Repo.preload(:hashtags) + |> Object.change(%{data: updated_object_data}) + |> maybe_touch_changeset(touch_changeset?) + + with {:ok, new_object} <- Repo.update(changeset), + {:ok, _} <- Object.invalid_object_cache(new_object), + {:ok, _} <- Object.set_cache(new_object), + # The metadata/utils.ex uses the object id for the cache. + {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(new_object.id) do + if used_history_in_new_object? do + with create_activity when not is_nil(create_activity) <- + Pleroma.Activity.get_create_by_object_ap_id(orig_object_ap_id), + {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(create_activity.id) do + nil + else + _ -> nil + end + end + + {:ok, new_object, updated} + end + end end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index e19642d50..098c177c7 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -428,37 +428,13 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do end if orig_object_data["type"] in Pleroma.Constants.updatable_object_types() do - %{ - updated_data: updated_object_data, - updated: updated, - used_history_in_new_object?: used_history_in_new_object? - } = Object.Updater.make_new_object_data_from_update_object(orig_object_data, updated_object) + {:ok, _, updated} = + Object.Updater.do_update_and_invalidate_cache(orig_object, updated_object) - changeset = - orig_object - |> Repo.preload(:hashtags) - |> Object.change(%{data: updated_object_data}) - - with {:ok, new_object} <- Repo.update(changeset), - {:ok, _} <- Object.invalid_object_cache(new_object), - {:ok, _} <- Object.set_cache(new_object), - # The metadata/utils.ex uses the object id for the cache. - {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(new_object.id) do - if used_history_in_new_object? do - with create_activity when not is_nil(create_activity) <- - Pleroma.Activity.get_create_by_object_ap_id(orig_object_ap_id), - {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(create_activity.id) do - nil - else - _ -> nil - end - end - - if updated do - object - |> Activity.normalize() - |> ActivityPub.notify_and_stream() - end + if updated do + object + |> Activity.normalize() + |> ActivityPub.notify_and_stream() end end diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs index c8ad66ddb..53c9277d6 100644 --- a/test/pleroma/object/fetcher_test.exs +++ b/test/pleroma/object/fetcher_test.exs @@ -9,8 +9,12 @@ defmodule Pleroma.Object.FetcherTest do alias Pleroma.Instances alias Pleroma.Object alias Pleroma.Object.Fetcher + alias Pleroma.Web.ActivityPub.ObjectValidator + + require Pleroma.Constants import Mock + import Pleroma.Factory import Tesla.Mock setup do @@ -284,6 +288,8 @@ defmodule Pleroma.Object.FetcherTest do describe "refetching" do setup do + insert(:user, ap_id: "https://mastodon.social/users/emelie") + object1 = %{ "id" => "https://mastodon.social/1", "actor" => "https://mastodon.social/users/emelie", @@ -293,10 +299,14 @@ defmodule Pleroma.Object.FetcherTest do "bcc" => [], "bto" => [], "cc" => [], - "to" => [], - "summary" => "" + "to" => [Pleroma.Constants.as_public()], + "summary" => "", + "published" => "2023-05-08 23:43:20Z", + "updated" => "2023-05-09 23:43:20Z" } + {:ok, local_object1, _} = ObjectValidator.validate(object1, []) + object2 = %{ "id" => "https://mastodon.social/2", "actor" => "https://mastodon.social/users/emelie", @@ -306,8 +316,10 @@ defmodule Pleroma.Object.FetcherTest do "bcc" => [], "bto" => [], "cc" => [], - "to" => [], + "to" => [Pleroma.Constants.as_public()], "summary" => "", + "published" => "2023-05-08 23:43:20Z", + "updated" => "2023-05-09 23:43:25Z", "formerRepresentations" => %{ "type" => "OrderedCollection", "orderedItems" => [ @@ -319,14 +331,18 @@ defmodule Pleroma.Object.FetcherTest do "bcc" => [], "bto" => [], "cc" => [], - "to" => [], - "summary" => "" + "to" => [Pleroma.Constants.as_public()], + "summary" => "", + "published" => "2023-05-08 23:43:20Z", + "updated" => "2023-05-09 23:43:21Z" } ], "totalItems" => 1 } } + {:ok, local_object2, _} = ObjectValidator.validate(object2, []) + mock(fn %{ method: :get, @@ -335,7 +351,7 @@ defmodule Pleroma.Object.FetcherTest do %Tesla.Env{ status: 200, headers: [{"content-type", "application/activity+json"}], - body: Jason.encode!(object1) + body: Jason.encode!(object1 |> Map.put("updated", "2023-05-09 23:44:20Z")) } %{ @@ -345,7 +361,7 @@ defmodule Pleroma.Object.FetcherTest do %Tesla.Env{ status: 200, headers: [{"content-type", "application/activity+json"}], - body: Jason.encode!(object2) + body: Jason.encode!(object2 |> Map.put("updated", "2023-05-09 23:44:20Z")) } %{ @@ -370,7 +386,7 @@ defmodule Pleroma.Object.FetcherTest do apply(HttpRequestMock, :request, [env]) end) - %{object1: object1, object2: object2} + %{object1: local_object1, object2: local_object2} end test "it keeps formerRepresentations if remote does not have this attr", %{object1: object1} do @@ -388,8 +404,9 @@ defmodule Pleroma.Object.FetcherTest do "bcc" => [], "bto" => [], "cc" => [], - "to" => [], - "summary" => "" + "to" => [Pleroma.Constants.as_public()], + "summary" => "", + "published" => "2023-05-08 23:43:20Z" } ], "totalItems" => 1 @@ -467,6 +484,53 @@ defmodule Pleroma.Object.FetcherTest do } } = refetched.data end + + test "it keeps the history intact if only updated time has changed", + %{object1: object1} do + full_object1 = + object1 + |> Map.merge(%{ + "updated" => "2023-05-08 23:43:47Z", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{"type" => "Note", "content" => "mew mew 1"} + ], + "totalItems" => 1 + } + }) + + {:ok, o} = Object.create(full_object1) + + assert {:ok, refetched} = Fetcher.refetch_object(o) + + assert %{ + "content" => "test 1", + "formerRepresentations" => %{ + "orderedItems" => [ + %{"content" => "mew mew 1"} + ], + "totalItems" => 1 + } + } = refetched.data + end + + test "it goes through ObjectValidator and MRF", %{object2: object2} do + with_mock Pleroma.Web.ActivityPub.MRF, [:passthrough], + filter: fn + %{"type" => "Note"} = object -> + {:ok, Map.put(object, "content", "MRFd content")} + + arg -> + passthrough([arg]) + end do + {:ok, o} = Object.create(object2) + + assert {:ok, refetched} = Fetcher.refetch_object(o) + + assert %{"content" => "MRFd content"} = refetched.data + end + end end describe "fetch with history" do From e170fc40dd46b2cfa4970c5d8174bdcfd6cf821c Mon Sep 17 00:00:00 2001 From: tusooa Date: Tue, 9 May 2023 21:38:28 -0400 Subject: [PATCH 54/89] Fix build warning --- lib/pleroma/object/updater.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex index 0c523ed35..bad811965 100644 --- a/lib/pleroma/object/updater.ex +++ b/lib/pleroma/object/updater.ex @@ -253,7 +253,6 @@ defmodule Pleroma.Object.Updater do def do_update_and_invalidate_cache(orig_object, updated_object, touch_changeset? \\ false) do orig_object_ap_id = updated_object["id"] - orig_object = Object.get_by_ap_id(orig_object_ap_id) orig_object_data = orig_object.data %{ From fb3335ffe287205274167d73a3124d2f811f2b6b Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 17 May 2023 17:12:21 +0200 Subject: [PATCH 55/89] EctoType: Add BareUri --- .../object_validators/bare_uri.ex | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 lib/pleroma/ecto_type/activity_pub/object_validators/bare_uri.ex diff --git a/lib/pleroma/ecto_type/activity_pub/object_validators/bare_uri.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/bare_uri.ex new file mode 100644 index 000000000..1038296e7 --- /dev/null +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/bare_uri.ex @@ -0,0 +1,23 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2023 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.BareUri do + use Ecto.Type + + def type, do: :string + + def cast(uri) when is_binary(uri) do + case URI.parse(uri) do + %URI{scheme: nil} -> :error + %URI{} -> {:ok, uri} + _ -> :error + end + end + + def cast(_), do: :error + + def dump(data), do: {:ok, data} + + def load(data), do: {:ok, data} +end From a5066bb0789e15d808e99e8676c16ad74290419c Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 17 May 2023 17:13:26 +0200 Subject: [PATCH 56/89] CommonFields: Use BareUri for :url Closes: https://git.pleroma.social/pleroma/pleroma/-/issues/3121 --- changelog.d/3884.fix | 1 + .../object_validators/common_fields.ex | 2 +- .../object_validators/bare_uri_test.ex | 25 +++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 changelog.d/3884.fix create mode 100644 test/pleroma/ecto_type/activity_pub/object_validators/bare_uri_test.ex diff --git a/changelog.d/3884.fix b/changelog.d/3884.fix new file mode 100644 index 000000000..f8dbb2bbf --- /dev/null +++ b/changelog.d/3884.fix @@ -0,0 +1 @@ +Allow non-HTTP(s) URIs in "url" fields for compatibility with "FEP-fffd: Proxy Objects" \ No newline at end of file diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex index 7b60c139a..d580208df 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex @@ -58,7 +58,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do field(:like_count, :integer, default: 0) field(:announcement_count, :integer, default: 0) field(:inReplyTo, ObjectValidators.ObjectID) - field(:url, ObjectValidators.Uri) + field(:url, ObjectValidators.BareUri) field(:likes, {:array, ObjectValidators.ObjectID}, default: []) field(:announcements, {:array, ObjectValidators.ObjectID}, default: []) diff --git a/test/pleroma/ecto_type/activity_pub/object_validators/bare_uri_test.ex b/test/pleroma/ecto_type/activity_pub/object_validators/bare_uri_test.ex new file mode 100644 index 000000000..226383c3c --- /dev/null +++ b/test/pleroma/ecto_type/activity_pub/object_validators/bare_uri_test.ex @@ -0,0 +1,25 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2023 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.BareUriTest do + use Pleroma.DataCase, async: true + + alias Pleroma.EctoType.ActivityPub.ObjectValidators.BareUri + + test "diaspora://" do + text = "diaspora://alice@fediverse.example/post/deadbeefdeadbeefdeadbeefdeadbeef" + assert {:ok, text} = BareUri.cast(text) + end + + test "nostr:" do + text = "nostr:note1gwdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + assert {:ok, text} = BareUri.cast(text) + end + + test "errors for non-URIs" do + assert :error == SafeText.cast(1) + assert :error == SafeText.cast("foo") + assert :error == SafeText.cast("foo bar") + end +end From 505e58d4eb84226613c85ac526c591dab5b1307e Mon Sep 17 00:00:00 2001 From: tusooa Date: Mon, 22 May 2023 08:14:20 -0400 Subject: [PATCH 57/89] Fix ObjectTest --- lib/pleroma/object/updater.ex | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex index bad811965..269609f1b 100644 --- a/lib/pleroma/object/updater.ex +++ b/lib/pleroma/object/updater.ex @@ -100,12 +100,14 @@ defmodule Pleroma.Object.Updater do end defp maybe_update_poll(to_be_updated, updated_object) do - choice_key = fn data -> - if Map.has_key?(data, "anyOf"), do: "anyOf", else: "oneOf" + choice_key = fn + %{"anyOf" => [_ | _]} = data -> "anyOf" + %{"oneOf" => [_ | _]} = data -> "oneOf" + _ -> nil end with true <- to_be_updated["type"] == "Question", - key <- choice_key.(updated_object), + key when not is_nil(key) <- choice_key.(updated_object), true <- key == choice_key.(to_be_updated), orig_choices <- to_be_updated[key] |> Enum.map(&Map.drop(&1, ["replies"])), new_choices <- updated_object[key] |> Enum.map(&Map.drop(&1, ["replies"])), From 6aafa7fe7661bcf9e62c16381ffeb2d32ec9bafa Mon Sep 17 00:00:00 2001 From: tusooa Date: Mon, 22 May 2023 08:16:14 -0400 Subject: [PATCH 58/89] Add changelog --- changelog.d/fix-object-test.fix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/fix-object-test.fix diff --git a/changelog.d/fix-object-test.fix b/changelog.d/fix-object-test.fix new file mode 100644 index 000000000..5eea719f0 --- /dev/null +++ b/changelog.d/fix-object-test.fix @@ -0,0 +1 @@ +Correctly handle the situation when a poll has both "anyOf" and "oneOf" but one of them being empty From 819a82da99588b53a67f42d8572a2484c4f17d40 Mon Sep 17 00:00:00 2001 From: tusooa Date: Mon, 22 May 2023 08:19:58 -0400 Subject: [PATCH 59/89] Fix unused variable --- lib/pleroma/object/updater.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex index 269609f1b..b1e4870ba 100644 --- a/lib/pleroma/object/updater.ex +++ b/lib/pleroma/object/updater.ex @@ -101,8 +101,8 @@ defmodule Pleroma.Object.Updater do defp maybe_update_poll(to_be_updated, updated_object) do choice_key = fn - %{"anyOf" => [_ | _]} = data -> "anyOf" - %{"oneOf" => [_ | _]} = data -> "oneOf" + %{"anyOf" => [_ | _]} -> "anyOf" + %{"oneOf" => [_ | _]} -> "oneOf" _ -> nil end From 38bcf6b19e3d83cb6c4e6c82d237a26edcab167a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 26 May 2023 12:08:50 -0400 Subject: [PATCH 60/89] MediaProxyController: Apply CSP sandbox --- changelog.d/3126.fix | 1 + .../web/media_proxy/media_proxy_controller.ex | 7 +++++++ .../media_proxy/media_proxy_controller_test.exs | 16 ++++++++++++++++ 3 files changed, 24 insertions(+) create mode 100644 changelog.d/3126.fix diff --git a/changelog.d/3126.fix b/changelog.d/3126.fix new file mode 100644 index 000000000..91d396c89 --- /dev/null +++ b/changelog.d/3126.fix @@ -0,0 +1 @@ +MediaProxy responses now return a sandbox CSP header diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index d2ad62c13..bda5b36ed 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -12,6 +12,8 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do alias Pleroma.Web.MediaProxy alias Plug.Conn + plug(:sandbox) + def remote(conn, %{"sig" => sig64, "url" => url64}) do with {_, true} <- {:enabled, MediaProxy.enabled?()}, {:ok, url} <- MediaProxy.decode_url(sig64, url64), @@ -202,4 +204,9 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do defp media_proxy_opts do Config.get([:media_proxy, :proxy_opts], []) end + + defp sandbox(conn, _params) do + conn + |> merge_resp_headers([{"content-security-policy", "sandbox;"}]) + end end diff --git a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs index 5246bf0c4..9ce092fd8 100644 --- a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs +++ b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs @@ -6,7 +6,9 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do use Pleroma.Web.ConnCase import Mock + import Mox + alias Pleroma.ReverseProxy.ClientMock alias Pleroma.Web.MediaProxy alias Plug.Conn @@ -74,6 +76,20 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do assert %Conn{status: 404, resp_body: "Not Found"} = get(conn, url) end end + + test "it applies sandbox CSP to MediaProxy requests", %{conn: conn} do + media_url = "https://lain.com/image.png" + media_proxy_url = MediaProxy.encode_url(media_url) + + ClientMock + |> expect(:request, fn :get, ^media_url, _, _, _ -> + {:ok, 200, [{"content-type", "image/png"}]} + end) + + %Conn{resp_headers: headers} = get(conn, media_proxy_url) + + assert {"content-security-policy", "sandbox;"} in headers + end end describe "Media Preview Proxy" do From 72833c84b583057ed0e757f35b65f90af40496df Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Wed, 17 May 2023 18:50:35 +0000 Subject: [PATCH 61/89] Merge branch 'tusooa/rework-refetch' into 'develop' Make sure object refetching follows update rules See merge request pleroma/pleroma!3883 --- changelog.d/3883.fix | 1 + lib/pleroma/object/fetcher.ex | 86 ++++---------------- lib/pleroma/object/updater.ex | 48 +++++++++++ lib/pleroma/web/activity_pub/side_effects.ex | 36 ++------ test/pleroma/object/fetcher_test.exs | 84 ++++++++++++++++--- 5 files changed, 144 insertions(+), 111 deletions(-) create mode 100644 changelog.d/3883.fix diff --git a/changelog.d/3883.fix b/changelog.d/3883.fix new file mode 100644 index 000000000..6824f2013 --- /dev/null +++ b/changelog.d/3883.fix @@ -0,0 +1 @@ +Fix abnormal behaviour when refetching a poll diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index a9a9eeeed..cc3772563 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -8,77 +8,30 @@ defmodule Pleroma.Object.Fetcher do alias Pleroma.Maps alias Pleroma.Object alias Pleroma.Object.Containment - alias Pleroma.Repo alias Pleroma.Signature alias Pleroma.Web.ActivityPub.InternalFetchActor + alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.ObjectValidator + alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.Federator require Logger require Pleroma.Constants - defp touch_changeset(changeset) do - updated_at = - NaiveDateTime.utc_now() - |> NaiveDateTime.truncate(:second) - - Ecto.Changeset.put_change(changeset, :updated_at, updated_at) - end - - defp maybe_reinject_internal_fields(%{data: %{} = old_data}, new_data) do - has_history? = fn - %{"formerRepresentations" => %{"orderedItems" => list}} when is_list(list) -> true - _ -> false - end - - internal_fields = Map.take(old_data, Pleroma.Constants.object_internal_fields()) - - remote_history_exists? = has_history?.(new_data) - - # If the remote history exists, we treat that as the only source of truth. - new_data = - if has_history?.(old_data) and not remote_history_exists? do - Map.put(new_data, "formerRepresentations", old_data["formerRepresentations"]) - else - new_data - end - - # If the remote does not have history information, we need to manage it ourselves - new_data = - if not remote_history_exists? do - changed? = - Pleroma.Constants.status_updatable_fields() - |> Enum.any?(fn field -> Map.get(old_data, field) != Map.get(new_data, field) end) - - %{updated_object: updated_object} = - new_data - |> Object.Updater.maybe_update_history(old_data, - updated: changed?, - use_history_in_new_object?: false - ) - - updated_object - else - new_data - end - - Map.merge(new_data, internal_fields) - end - - defp maybe_reinject_internal_fields(_, new_data), do: new_data - @spec reinject_object(struct(), map()) :: {:ok, Object.t()} | {:error, any()} - defp reinject_object(%Object{data: %{"type" => "Question"}} = object, new_data) do + defp reinject_object(%Object{data: %{}} = object, new_data) do Logger.debug("Reinjecting object #{new_data["id"]}") - with data <- maybe_reinject_internal_fields(object, new_data), - {:ok, data, _} <- ObjectValidator.validate(data, %{}), - changeset <- Object.change(object, %{data: data}), - changeset <- touch_changeset(changeset), - {:ok, object} <- Repo.insert_or_update(changeset), - {:ok, object} <- Object.set_cache(object) do - {:ok, object} + with {:ok, new_data, _} <- ObjectValidator.validate(new_data, %{}), + {:ok, new_data} <- MRF.filter(new_data), + {:ok, new_object, _} <- + Object.Updater.do_update_and_invalidate_cache( + object, + new_data, + _touch_changeset? = true + ) do + {:ok, new_object} else e -> Logger.error("Error while processing object: #{inspect(e)}") @@ -86,20 +39,11 @@ defmodule Pleroma.Object.Fetcher do end end - defp reinject_object(%Object{} = object, new_data) do - Logger.debug("Reinjecting object #{new_data["id"]}") - - with new_data <- Transmogrifier.fix_object(new_data), - data <- maybe_reinject_internal_fields(object, new_data), - changeset <- Object.change(object, %{data: data}), - changeset <- touch_changeset(changeset), - {:ok, object} <- Repo.insert_or_update(changeset), - {:ok, object} <- Object.set_cache(object) do + defp reinject_object(_, new_data) do + with {:ok, object, _} <- Pipeline.common_pipeline(new_data, local: false) do {:ok, object} else - e -> - Logger.error("Error while processing object: #{inspect(e)}") - {:error, e} + e -> e end end diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex index ab38d3ed2..bad811965 100644 --- a/lib/pleroma/object/updater.ex +++ b/lib/pleroma/object/updater.ex @@ -5,6 +5,9 @@ defmodule Pleroma.Object.Updater do require Pleroma.Constants + alias Pleroma.Object + alias Pleroma.Repo + def update_content_fields(orig_object_data, updated_object) do Pleroma.Constants.status_updatable_fields() |> Enum.reduce( @@ -237,4 +240,49 @@ defmodule Pleroma.Object.Updater do {:history_items, e} -> e end end + + defp maybe_touch_changeset(changeset, true) do + updated_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.truncate(:second) + + Ecto.Changeset.put_change(changeset, :updated_at, updated_at) + end + + defp maybe_touch_changeset(changeset, _), do: changeset + + def do_update_and_invalidate_cache(orig_object, updated_object, touch_changeset? \\ false) do + orig_object_ap_id = updated_object["id"] + orig_object_data = orig_object.data + + %{ + updated_data: updated_object_data, + updated: updated, + used_history_in_new_object?: used_history_in_new_object? + } = make_new_object_data_from_update_object(orig_object_data, updated_object) + + changeset = + orig_object + |> Repo.preload(:hashtags) + |> Object.change(%{data: updated_object_data}) + |> maybe_touch_changeset(touch_changeset?) + + with {:ok, new_object} <- Repo.update(changeset), + {:ok, _} <- Object.invalid_object_cache(new_object), + {:ok, _} <- Object.set_cache(new_object), + # The metadata/utils.ex uses the object id for the cache. + {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(new_object.id) do + if used_history_in_new_object? do + with create_activity when not is_nil(create_activity) <- + Pleroma.Activity.get_create_by_object_ap_id(orig_object_ap_id), + {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(create_activity.id) do + nil + else + _ -> nil + end + end + + {:ok, new_object, updated} + end + end end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index a2152b945..fc5dec362 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -428,37 +428,13 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do end if orig_object_data["type"] in Pleroma.Constants.updatable_object_types() do - %{ - updated_data: updated_object_data, - updated: updated, - used_history_in_new_object?: used_history_in_new_object? - } = Object.Updater.make_new_object_data_from_update_object(orig_object_data, updated_object) + {:ok, _, updated} = + Object.Updater.do_update_and_invalidate_cache(orig_object, updated_object) - changeset = - orig_object - |> Repo.preload(:hashtags) - |> Object.change(%{data: updated_object_data}) - - with {:ok, new_object} <- Repo.update(changeset), - {:ok, _} <- Object.invalid_object_cache(new_object), - {:ok, _} <- Object.set_cache(new_object), - # The metadata/utils.ex uses the object id for the cache. - {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(new_object.id) do - if used_history_in_new_object? do - with create_activity when not is_nil(create_activity) <- - Pleroma.Activity.get_create_by_object_ap_id(orig_object_ap_id), - {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(create_activity.id) do - nil - else - _ -> nil - end - end - - if updated do - object - |> Activity.normalize() - |> ActivityPub.notify_and_stream() - end + if updated do + object + |> Activity.normalize() + |> ActivityPub.notify_and_stream() end end diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs index c8ad66ddb..53c9277d6 100644 --- a/test/pleroma/object/fetcher_test.exs +++ b/test/pleroma/object/fetcher_test.exs @@ -9,8 +9,12 @@ defmodule Pleroma.Object.FetcherTest do alias Pleroma.Instances alias Pleroma.Object alias Pleroma.Object.Fetcher + alias Pleroma.Web.ActivityPub.ObjectValidator + + require Pleroma.Constants import Mock + import Pleroma.Factory import Tesla.Mock setup do @@ -284,6 +288,8 @@ defmodule Pleroma.Object.FetcherTest do describe "refetching" do setup do + insert(:user, ap_id: "https://mastodon.social/users/emelie") + object1 = %{ "id" => "https://mastodon.social/1", "actor" => "https://mastodon.social/users/emelie", @@ -293,10 +299,14 @@ defmodule Pleroma.Object.FetcherTest do "bcc" => [], "bto" => [], "cc" => [], - "to" => [], - "summary" => "" + "to" => [Pleroma.Constants.as_public()], + "summary" => "", + "published" => "2023-05-08 23:43:20Z", + "updated" => "2023-05-09 23:43:20Z" } + {:ok, local_object1, _} = ObjectValidator.validate(object1, []) + object2 = %{ "id" => "https://mastodon.social/2", "actor" => "https://mastodon.social/users/emelie", @@ -306,8 +316,10 @@ defmodule Pleroma.Object.FetcherTest do "bcc" => [], "bto" => [], "cc" => [], - "to" => [], + "to" => [Pleroma.Constants.as_public()], "summary" => "", + "published" => "2023-05-08 23:43:20Z", + "updated" => "2023-05-09 23:43:25Z", "formerRepresentations" => %{ "type" => "OrderedCollection", "orderedItems" => [ @@ -319,14 +331,18 @@ defmodule Pleroma.Object.FetcherTest do "bcc" => [], "bto" => [], "cc" => [], - "to" => [], - "summary" => "" + "to" => [Pleroma.Constants.as_public()], + "summary" => "", + "published" => "2023-05-08 23:43:20Z", + "updated" => "2023-05-09 23:43:21Z" } ], "totalItems" => 1 } } + {:ok, local_object2, _} = ObjectValidator.validate(object2, []) + mock(fn %{ method: :get, @@ -335,7 +351,7 @@ defmodule Pleroma.Object.FetcherTest do %Tesla.Env{ status: 200, headers: [{"content-type", "application/activity+json"}], - body: Jason.encode!(object1) + body: Jason.encode!(object1 |> Map.put("updated", "2023-05-09 23:44:20Z")) } %{ @@ -345,7 +361,7 @@ defmodule Pleroma.Object.FetcherTest do %Tesla.Env{ status: 200, headers: [{"content-type", "application/activity+json"}], - body: Jason.encode!(object2) + body: Jason.encode!(object2 |> Map.put("updated", "2023-05-09 23:44:20Z")) } %{ @@ -370,7 +386,7 @@ defmodule Pleroma.Object.FetcherTest do apply(HttpRequestMock, :request, [env]) end) - %{object1: object1, object2: object2} + %{object1: local_object1, object2: local_object2} end test "it keeps formerRepresentations if remote does not have this attr", %{object1: object1} do @@ -388,8 +404,9 @@ defmodule Pleroma.Object.FetcherTest do "bcc" => [], "bto" => [], "cc" => [], - "to" => [], - "summary" => "" + "to" => [Pleroma.Constants.as_public()], + "summary" => "", + "published" => "2023-05-08 23:43:20Z" } ], "totalItems" => 1 @@ -467,6 +484,53 @@ defmodule Pleroma.Object.FetcherTest do } } = refetched.data end + + test "it keeps the history intact if only updated time has changed", + %{object1: object1} do + full_object1 = + object1 + |> Map.merge(%{ + "updated" => "2023-05-08 23:43:47Z", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{"type" => "Note", "content" => "mew mew 1"} + ], + "totalItems" => 1 + } + }) + + {:ok, o} = Object.create(full_object1) + + assert {:ok, refetched} = Fetcher.refetch_object(o) + + assert %{ + "content" => "test 1", + "formerRepresentations" => %{ + "orderedItems" => [ + %{"content" => "mew mew 1"} + ], + "totalItems" => 1 + } + } = refetched.data + end + + test "it goes through ObjectValidator and MRF", %{object2: object2} do + with_mock Pleroma.Web.ActivityPub.MRF, [:passthrough], + filter: fn + %{"type" => "Note"} = object -> + {:ok, Map.put(object, "content", "MRFd content")} + + arg -> + passthrough([arg]) + end do + {:ok, o} = Object.create(object2) + + assert {:ok, refetched} = Fetcher.refetch_object(o) + + assert %{"content" => "MRFd content"} = refetched.data + end + end end describe "fetch with history" do From 4339230f64b05fee1c4d7313c1dc9adc45827a5d Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Tue, 23 May 2023 01:57:07 +0000 Subject: [PATCH 62/89] Merge branch 'tusooa/fix-object-test' into 'develop' Fix ObjectTest See merge request pleroma/pleroma!3887 --- changelog.d/fix-object-test.fix | 1 + lib/pleroma/object/updater.ex | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 changelog.d/fix-object-test.fix diff --git a/changelog.d/fix-object-test.fix b/changelog.d/fix-object-test.fix new file mode 100644 index 000000000..5eea719f0 --- /dev/null +++ b/changelog.d/fix-object-test.fix @@ -0,0 +1 @@ +Correctly handle the situation when a poll has both "anyOf" and "oneOf" but one of them being empty diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex index bad811965..b1e4870ba 100644 --- a/lib/pleroma/object/updater.ex +++ b/lib/pleroma/object/updater.ex @@ -100,12 +100,14 @@ defmodule Pleroma.Object.Updater do end defp maybe_update_poll(to_be_updated, updated_object) do - choice_key = fn data -> - if Map.has_key?(data, "anyOf"), do: "anyOf", else: "oneOf" + choice_key = fn + %{"anyOf" => [_ | _]} -> "anyOf" + %{"oneOf" => [_ | _]} -> "oneOf" + _ -> nil end with true <- to_be_updated["type"] == "Question", - key <- choice_key.(updated_object), + key when not is_nil(key) <- choice_key.(updated_object), true <- key == choice_key.(to_be_updated), orig_choices <- to_be_updated[key] |> Enum.map(&Map.drop(&1, ["replies"])), new_choices <- updated_object[key] |> Enum.map(&Map.drop(&1, ["replies"])), From b36263e5ffd0d89d819b01478f19891b14740bb0 Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Fri, 26 May 2023 17:12:18 +0000 Subject: [PATCH 63/89] Merge branch 'issue/3126' into 'develop' MediaProxyController: Apply CSP sandbox See merge request pleroma/pleroma!3890 --- changelog.d/3126.fix | 1 + .../web/media_proxy/media_proxy_controller.ex | 7 +++++++ .../media_proxy/media_proxy_controller_test.exs | 16 ++++++++++++++++ 3 files changed, 24 insertions(+) create mode 100644 changelog.d/3126.fix diff --git a/changelog.d/3126.fix b/changelog.d/3126.fix new file mode 100644 index 000000000..91d396c89 --- /dev/null +++ b/changelog.d/3126.fix @@ -0,0 +1 @@ +MediaProxy responses now return a sandbox CSP header diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index d2ad62c13..bda5b36ed 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -12,6 +12,8 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do alias Pleroma.Web.MediaProxy alias Plug.Conn + plug(:sandbox) + def remote(conn, %{"sig" => sig64, "url" => url64}) do with {_, true} <- {:enabled, MediaProxy.enabled?()}, {:ok, url} <- MediaProxy.decode_url(sig64, url64), @@ -202,4 +204,9 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do defp media_proxy_opts do Config.get([:media_proxy, :proxy_opts], []) end + + defp sandbox(conn, _params) do + conn + |> merge_resp_headers([{"content-security-policy", "sandbox;"}]) + end end diff --git a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs index 5246bf0c4..9ce092fd8 100644 --- a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs +++ b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs @@ -6,7 +6,9 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do use Pleroma.Web.ConnCase import Mock + import Mox + alias Pleroma.ReverseProxy.ClientMock alias Pleroma.Web.MediaProxy alias Plug.Conn @@ -74,6 +76,20 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do assert %Conn{status: 404, resp_body: "Not Found"} = get(conn, url) end end + + test "it applies sandbox CSP to MediaProxy requests", %{conn: conn} do + media_url = "https://lain.com/image.png" + media_proxy_url = MediaProxy.encode_url(media_url) + + ClientMock + |> expect(:request, fn :get, ^media_url, _, _, _ -> + {:ok, 200, [{"content-type", "image/png"}]} + end) + + %Conn{resp_headers: headers} = get(conn, media_proxy_url) + + assert {"content-security-policy", "sandbox;"} in headers + end end describe "Media Preview Proxy" do From d0c2e0830b16c541d2883004f071a0954be45682 Mon Sep 17 00:00:00 2001 From: tusooa Date: Thu, 30 Mar 2023 21:01:37 -0400 Subject: [PATCH 64/89] Enforce unauth restrictions for public streaming endpoints --- lib/pleroma/web/streamer.ex | 46 +++++++++++++++--- test/pleroma/web/streamer_test.exs | 77 ++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 3c0da5c27..b9a04cc76 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -25,6 +25,7 @@ defmodule Pleroma.Web.Streamer do def registry, do: @registry @public_streams ["public", "public:local", "public:media", "public:local:media"] + @local_streams ["public:local", "public:local:media"] @user_streams ["user", "user:notification", "direct", "user:pleroma_chat"] @doc "Expands and authorizes a stream, and registers the process for streaming." @@ -41,14 +42,37 @@ defmodule Pleroma.Web.Streamer do end end + defp can_access_stream(user, oauth_token, kind) do + with {_, true} <- {:restrict?, Config.restrict_unauthenticated_access?(:timelines, kind)}, + {_, %User{id: user_id}, %Token{user_id: user_id}} <- {:user, user, oauth_token}, + {_, true} <- + {:scopes, + OAuthScopesPlug.filter_descendants(["read:statuses"], oauth_token.scopes) != []} do + true + else + {:restrict?, _} -> + true + + _ -> + false + end + end + @doc "Expand and authorizes a stream" @spec get_topic(stream :: String.t(), User.t() | nil, Token.t() | nil, Map.t()) :: {:ok, topic :: String.t()} | {:error, :bad_topic} def get_topic(stream, user, oauth_token, params \\ %{}) - # Allow all public steams. - def get_topic(stream, _user, _oauth_token, _params) when stream in @public_streams do - {:ok, stream} + # Allow all public steams if the instance allows unauthenticated access. + # Otherwise, only allow users with valid oauth tokens. + def get_topic(stream, user, oauth_token, _params) when stream in @public_streams do + kind = if stream in @local_streams, do: :local, else: :federated + + if can_access_stream(user, oauth_token, kind) do + {:ok, stream} + else + {:error, :unauthorized} + end end # Allow all hashtags streams. @@ -57,12 +81,20 @@ defmodule Pleroma.Web.Streamer do end # Allow remote instance streams. - def get_topic("public:remote", _user, _oauth_token, %{"instance" => instance} = _params) do - {:ok, "public:remote:" <> instance} + def get_topic("public:remote", user, oauth_token, %{"instance" => instance} = _params) do + if can_access_stream(user, oauth_token, :federated) do + {:ok, "public:remote:" <> instance} + else + {:error, :unauthorized} + end end - def get_topic("public:remote:media", _user, _oauth_token, %{"instance" => instance} = _params) do - {:ok, "public:remote:media:" <> instance} + def get_topic("public:remote:media", user, oauth_token, %{"instance" => instance} = _params) do + if can_access_stream(user, oauth_token, :federated) do + {:ok, "public:remote:media:" <> instance} + else + {:error, :unauthorized} + end end # Expand user streams. diff --git a/test/pleroma/web/streamer_test.exs b/test/pleroma/web/streamer_test.exs index 8b0c84164..7ab0e379b 100644 --- a/test/pleroma/web/streamer_test.exs +++ b/test/pleroma/web/streamer_test.exs @@ -29,6 +29,26 @@ defmodule Pleroma.Web.StreamerTest do assert {:ok, "public:local:media"} = Streamer.get_topic("public:local:media", nil, nil) end + test "rejects local public streams if restricted_unauthenticated is on" do + clear_config([:restrict_unauthenticated, :timelines, :local], true) + + assert {:error, :unauthorized} = Streamer.get_topic("public:local", nil, nil) + assert {:error, :unauthorized} = Streamer.get_topic("public:local:media", nil, nil) + end + + test "rejects remote public streams if restricted_unauthenticated is on" do + clear_config([:restrict_unauthenticated, :timelines, :federated], true) + + assert {:error, :unauthorized} = Streamer.get_topic("public", nil, nil) + assert {:error, :unauthorized} = Streamer.get_topic("public:media", nil, nil) + + assert {:error, :unauthorized} = + Streamer.get_topic("public:remote", nil, nil, %{"instance" => "lain.com"}) + + assert {:error, :unauthorized} = + Streamer.get_topic("public:remote:media", nil, nil, %{"instance" => "lain.com"}) + end + test "allows instance streams" do assert {:ok, "public:remote:lain.com"} = Streamer.get_topic("public:remote", nil, nil, %{"instance" => "lain.com"}) @@ -69,6 +89,63 @@ defmodule Pleroma.Web.StreamerTest do end end + test "allows local public streams if restricted_unauthenticated is on", %{ + user: user, + token: oauth_token + } do + clear_config([:restrict_unauthenticated, :timelines, :local], true) + + %{token: read_notifications_token} = oauth_access(["read:notifications"], user: user) + %{token: badly_scoped_token} = oauth_access(["irrelevant:scope"], user: user) + + assert {:ok, "public:local"} = Streamer.get_topic("public:local", user, oauth_token) + + assert {:ok, "public:local:media"} = + Streamer.get_topic("public:local:media", user, oauth_token) + + for token <- [read_notifications_token, badly_scoped_token] do + assert {:error, :unauthorized} = Streamer.get_topic("public:local", user, token) + + assert {:error, :unauthorized} = Streamer.get_topic("public:local:media", user, token) + end + end + + test "allows remote public streams if restricted_unauthenticated is on", %{ + user: user, + token: oauth_token + } do + clear_config([:restrict_unauthenticated, :timelines, :federated], true) + + %{token: read_notifications_token} = oauth_access(["read:notifications"], user: user) + %{token: badly_scoped_token} = oauth_access(["irrelevant:scope"], user: user) + + assert {:ok, "public"} = Streamer.get_topic("public", user, oauth_token) + assert {:ok, "public:media"} = Streamer.get_topic("public:media", user, oauth_token) + + assert {:ok, "public:remote:lain.com"} = + Streamer.get_topic("public:remote", user, oauth_token, %{"instance" => "lain.com"}) + + assert {:ok, "public:remote:media:lain.com"} = + Streamer.get_topic("public:remote:media", user, oauth_token, %{ + "instance" => "lain.com" + }) + + for token <- [read_notifications_token, badly_scoped_token] do + assert {:error, :unauthorized} = Streamer.get_topic("public", user, token) + assert {:error, :unauthorized} = Streamer.get_topic("public:media", user, token) + + assert {:error, :unauthorized} = + Streamer.get_topic("public:remote", user, token, %{ + "instance" => "lain.com" + }) + + assert {:error, :unauthorized} = + Streamer.get_topic("public:remote:media", user, token, %{ + "instance" => "lain.com" + }) + end + end + test "allows user streams (with proper OAuth token scopes)", %{ user: user, token: read_oauth_token From 0d68804aa7efc4f3212e02218804755da93d03f0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 26 May 2023 13:23:59 -0400 Subject: [PATCH 65/89] Filter OEmbed HTML tags --- changelog.d/3891.fix | 1 + lib/pleroma/web/rich_media/parsers/o_embed.ex | 4 ++-- test/pleroma/web/rich_media/parser_test.exs | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 changelog.d/3891.fix diff --git a/changelog.d/3891.fix b/changelog.d/3891.fix new file mode 100644 index 000000000..f1fb62d82 --- /dev/null +++ b/changelog.d/3891.fix @@ -0,0 +1 @@ +OEmbed HTML tags are now filtered diff --git a/lib/pleroma/web/rich_media/parsers/o_embed.ex b/lib/pleroma/web/rich_media/parsers/o_embed.ex index 75318d9c7..0f303176c 100644 --- a/lib/pleroma/web/rich_media/parsers/o_embed.ex +++ b/lib/pleroma/web/rich_media/parsers/o_embed.ex @@ -6,8 +6,8 @@ defmodule Pleroma.Web.RichMedia.Parsers.OEmbed do def parse(html, _data) do with elements = [_ | _] <- get_discovery_data(html), oembed_url when is_binary(oembed_url) <- get_oembed_url(elements), - {:ok, oembed_data} <- get_oembed_data(oembed_url) do - oembed_data + {:ok, oembed_data = %{"html" => html}} <- get_oembed_data(oembed_url) do + %{oembed_data | "html" => Pleroma.HTML.filter_tags(html)} else _e -> %{} end diff --git a/test/pleroma/web/rich_media/parser_test.exs b/test/pleroma/web/rich_media/parser_test.exs index ffdc4e5d7..9064138a6 100644 --- a/test/pleroma/web/rich_media/parser_test.exs +++ b/test/pleroma/web/rich_media/parser_test.exs @@ -129,7 +129,7 @@ defmodule Pleroma.Web.RichMedia.ParserTest do }} end - test "parses OEmbed" do + test "parses OEmbed and filters HTML tags" do assert Parser.parse("http://example.com/oembed") == {:ok, %{ @@ -139,7 +139,7 @@ defmodule Pleroma.Web.RichMedia.ParserTest do "flickr_type" => "photo", "height" => "768", "html" => - "\"Bacon", + "\"Bacon", "license" => "All Rights Reserved", "license_id" => 0, "provider_name" => "Flickr", From 4505bc1e587ce76683e21a41f90fb2956ed2a084 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 26 May 2023 13:23:59 -0400 Subject: [PATCH 66/89] Filter OEmbed HTML tags --- changelog.d/3891.fix | 1 + lib/pleroma/web/rich_media/parsers/o_embed.ex | 4 ++-- test/pleroma/web/rich_media/parser_test.exs | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 changelog.d/3891.fix diff --git a/changelog.d/3891.fix b/changelog.d/3891.fix new file mode 100644 index 000000000..f1fb62d82 --- /dev/null +++ b/changelog.d/3891.fix @@ -0,0 +1 @@ +OEmbed HTML tags are now filtered diff --git a/lib/pleroma/web/rich_media/parsers/o_embed.ex b/lib/pleroma/web/rich_media/parsers/o_embed.ex index 75318d9c7..0f303176c 100644 --- a/lib/pleroma/web/rich_media/parsers/o_embed.ex +++ b/lib/pleroma/web/rich_media/parsers/o_embed.ex @@ -6,8 +6,8 @@ defmodule Pleroma.Web.RichMedia.Parsers.OEmbed do def parse(html, _data) do with elements = [_ | _] <- get_discovery_data(html), oembed_url when is_binary(oembed_url) <- get_oembed_url(elements), - {:ok, oembed_data} <- get_oembed_data(oembed_url) do - oembed_data + {:ok, oembed_data = %{"html" => html}} <- get_oembed_data(oembed_url) do + %{oembed_data | "html" => Pleroma.HTML.filter_tags(html)} else _e -> %{} end diff --git a/test/pleroma/web/rich_media/parser_test.exs b/test/pleroma/web/rich_media/parser_test.exs index ffdc4e5d7..9064138a6 100644 --- a/test/pleroma/web/rich_media/parser_test.exs +++ b/test/pleroma/web/rich_media/parser_test.exs @@ -129,7 +129,7 @@ defmodule Pleroma.Web.RichMedia.ParserTest do }} end - test "parses OEmbed" do + test "parses OEmbed and filters HTML tags" do assert Parser.parse("http://example.com/oembed") == {:ok, %{ @@ -139,7 +139,7 @@ defmodule Pleroma.Web.RichMedia.ParserTest do "flickr_type" => "photo", "height" => "768", "html" => - "\"Bacon", + "\"Bacon", "license" => "All Rights Reserved", "license_id" => 0, "provider_name" => "Flickr", From 7618e562b34d4240514d8819407dd6bd3adbe79d Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 30 Mar 2023 12:52:57 +0200 Subject: [PATCH 67/89] Version 2.5.2 --- CHANGELOG.md | 19 +++++++++++++++++++ mix.exs | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1742ee2f4..f6fc6aaee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,25 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed +### Removed + +## 2.5.2 + +### Security +- `/proxy` endpoint now sets a Content-Security-Policy (sandbox) +- WebSocket endpoint now respects unauthenticated restrictions for streams of public posts +- OEmbed HTML tags are now filtered + +### Changed +- docs: Be more explicit about the level of compatibility of OTP releases +- Set default background worker timeout to 15 minutes + +### Fixed +- Atom/RSS formatting (HTML truncation, published, missing summary) +- Remove `static_fe` pipeline for `/users/:nickname/feed` +- Stop oban from retrying if validating errors occur when processing incoming data +- Make sure object refetching as used by already received polls follows MRF rules + ### Removed - BREAKING: Support for passwords generated with `crypt(3)` (Gnu Social migration artifact) diff --git a/mix.exs b/mix.exs index 7439bbca6..79fd9c9ef 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do def project do [ app: :pleroma, - version: version("2.5.1"), + version: version("2.5.2"), elixir: "~> 1.11", elixirc_paths: elixirc_paths(Mix.env()), compilers: [:phoenix, :gettext] ++ Mix.compilers(), From b6b7de20100e980849404a9e7384617a15df1e43 Mon Sep 17 00:00:00 2001 From: faried nawaz Date: Thu, 18 May 2023 02:12:20 +0500 Subject: [PATCH 68/89] add url to Metadata.build_tags call If static_fe is enabled, going to https://pleroma/notice/some-id results in With this fix, it is Additionally, Pleroma.Web.Metadata.Providers.OpenGraph now generates meta tags for attachments in the post. --- lib/pleroma/web/static_fe/static_fe_controller.ex | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index 97c41c6f9..8019a218a 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -25,7 +25,15 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do true <- Visibility.is_public?(activity.object), {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)}, %User{} = user <- User.get_by_ap_id(activity.object.data["actor"]) do - meta = Metadata.build_tags(%{activity_id: notice_id, object: activity.object, user: user}) + url = Helpers.url(conn) <> conn.request_path + + meta = + Metadata.build_tags(%{ + activity_id: notice_id, + object: activity.object, + user: user, + url: url + }) timeline = activity.object.data["context"] From 52368e670222c2add5f5677d864dd64c9376c553 Mon Sep 17 00:00:00 2001 From: faried nawaz Date: Thu, 18 May 2023 02:17:29 +0500 Subject: [PATCH 69/89] fix meta tag for twitter cards and image attachments The name of the tag should be twitter:image, not twitter:player. Also, add twitter:image:alt meta tags. --- lib/pleroma/web/metadata/providers/twitter_card.ex | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/metadata/providers/twitter_card.ex b/lib/pleroma/web/metadata/providers/twitter_card.ex index 2dac22ee2..fbd4fd5a7 100644 --- a/lib/pleroma/web/metadata/providers/twitter_card.ex +++ b/lib/pleroma/web/metadata/providers/twitter_card.ex @@ -76,9 +76,11 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCard do {:meta, [name: "twitter:card", content: "summary_large_image"], []}, {:meta, [ - name: "twitter:player", + name: "twitter:image", content: MediaProxy.url(url["href"]) - ], []} + ], []}, + {:meta, [property: "twitter:image:alt", content: truncate(attachment["name"])], + []} | acc ] |> maybe_add_dimensions(url) @@ -130,4 +132,10 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCard do metadata end end + + defp truncate(text) do + # truncate to 420 characters + # see https://developer.twitter.com/en/docs/twitter-for-websites/cards/overview/markup + Pleroma.Formatter.truncate(text, 420) + end end From a1af122499d3558d9fa6d5606c394e01f36e95e4 Mon Sep 17 00:00:00 2001 From: faried nawaz Date: Thu, 18 May 2023 02:28:50 +0500 Subject: [PATCH 70/89] changelog entry --- changelog.d/3885.fix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3885.fix diff --git a/changelog.d/3885.fix b/changelog.d/3885.fix new file mode 100644 index 000000000..c5fbb0ed4 --- /dev/null +++ b/changelog.d/3885.fix @@ -0,0 +1 @@ +Fix opengraph and twitter card meta tags From 8b390d27dce0c1827ed4c8c959612b6126624c9a Mon Sep 17 00:00:00 2001 From: faried nawaz Date: Thu, 18 May 2023 03:20:44 +0500 Subject: [PATCH 71/89] twitter card: handle case where image has no alt text --- lib/pleroma/web/metadata/providers/twitter_card.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/metadata/providers/twitter_card.ex b/lib/pleroma/web/metadata/providers/twitter_card.ex index fbd4fd5a7..426022c65 100644 --- a/lib/pleroma/web/metadata/providers/twitter_card.ex +++ b/lib/pleroma/web/metadata/providers/twitter_card.ex @@ -79,8 +79,7 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCard do name: "twitter:image", content: MediaProxy.url(url["href"]) ], []}, - {:meta, [property: "twitter:image:alt", content: truncate(attachment["name"])], - []} + {:meta, [name: "twitter:image:alt", content: truncate(attachment["name"])], []} | acc ] |> maybe_add_dimensions(url) @@ -133,6 +132,8 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCard do end end + defp truncate(nil), do: "" + defp truncate(text) do # truncate to 420 characters # see https://developer.twitter.com/en/docs/twitter-for-websites/cards/overview/markup From 4c91c0d1baa5bdfdcbfc6ff54859be7e9db91a37 Mon Sep 17 00:00:00 2001 From: faried nawaz Date: Thu, 18 May 2023 03:21:14 +0500 Subject: [PATCH 72/89] oops, forgot the test cases --- test/pleroma/web/metadata/providers/twitter_card_test.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/pleroma/web/metadata/providers/twitter_card_test.exs b/test/pleroma/web/metadata/providers/twitter_card_test.exs index be4cfbe7b..f8d01c5c8 100644 --- a/test/pleroma/web/metadata/providers/twitter_card_test.exs +++ b/test/pleroma/web/metadata/providers/twitter_card_test.exs @@ -182,7 +182,8 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCardTest do {:meta, [name: "twitter:title", content: Utils.user_name_string(user)], []}, {:meta, [name: "twitter:description", content: "pleroma in a nutshell"], []}, {:meta, [name: "twitter:card", content: "summary_large_image"], []}, - {:meta, [name: "twitter:player", content: "https://pleroma.gov/tenshi.png"], []}, + {:meta, [name: "twitter:image", content: "https://pleroma.gov/tenshi.png"], []}, + {:meta, [name: "twitter:image:alt", content: ""], []}, {:meta, [name: "twitter:player:width", content: "1280"], []}, {:meta, [name: "twitter:player:height", content: "1024"], []}, {:meta, [name: "twitter:card", content: "player"], []}, From 506a1c98e716754455387be9ace3ad7aec9c47a3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 29 May 2023 13:55:48 -0400 Subject: [PATCH 73/89] ConnCase: Make sure the host we use in tests is the actual Endpoint host --- test/support/conn_case.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/support/conn_case.ex b/test/support/conn_case.ex index f010fec33..d6729ab87 100644 --- a/test/support/conn_case.ex +++ b/test/support/conn_case.ex @@ -120,6 +120,10 @@ defmodule Pleroma.Web.ConnCase do Mox.verify_on_exit!() - {:ok, conn: Phoenix.ConnTest.build_conn()} + {:ok, + conn: + Phoenix.ConnTest.build_conn() + |> Map.put(:host, Pleroma.Web.Endpoint.host()) + |> Plug.Test.init_test_session(%{})} end end From 843fcca5b4d022e4c088d4a60839b4a286500148 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 29 May 2023 13:59:51 -0400 Subject: [PATCH 74/89] Validate Host header matches expected value before allowing access to MediaProxy --- .../web/media_proxy/media_proxy_controller.ex | 12 ++++++++++++ .../media_proxy/media_proxy_controller_test.exs | 17 +++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index bda5b36ed..767496b68 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -12,6 +12,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do alias Pleroma.Web.MediaProxy alias Plug.Conn + plug(:validate_host) plug(:sandbox) def remote(conn, %{"sig" => sig64, "url" => url64}) do @@ -205,6 +206,17 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do Config.get([:media_proxy, :proxy_opts], []) end + defp validate_host(conn, _params) do + proxy_host = MediaProxy.base_url() |> URI.parse() |> Map.get(:host) + + if match?(^proxy_host, conn.host) do + conn + else + send_resp(conn, 400, Conn.Status.reason_phrase(400)) + |> halt() + end + end + defp sandbox(conn, _params) do conn |> merge_resp_headers([{"content-security-policy", "sandbox;"}]) diff --git a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs index 9ce092fd8..3971330d4 100644 --- a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs +++ b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs @@ -54,6 +54,23 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do } = get(conn, "/proxy/hhgfh/eeee/fff") end + test "it returns a 400 for invalid host", %{conn: conn} do + clear_config([:media_proxy, :base_url], "http://mp.localhost/") + + url = + MediaProxy.encode_url("https://pleroma.social/logo.jpeg") + |> URI.parse() + |> Map.put(:host, "wronghost") + |> URI.to_string() + + with_mock Pleroma.ReverseProxy, + call: fn _conn, _url, _opts -> %Conn{status: :success} end do + %{status: status} = get(conn, url) + + assert status == 400 + end + end + test "redirects to valid url when filename is invalidated", %{conn: conn, url: url} do invalid_url = String.replace(url, "test.png", "test-file.png") response = get(conn, invalid_url) From a60dd0d92dae2471025827bc57d3cf3194003110 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 29 May 2023 14:16:03 -0400 Subject: [PATCH 75/89] Validate Host header matches expected value before allowing access to Uploads --- lib/pleroma/web/plugs/uploaded_media.ex | 11 ++++++++++- .../pleroma/web/plugs/uploaded_media_plug_test.exs | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/plugs/uploaded_media.ex b/lib/pleroma/web/plugs/uploaded_media.ex index 8b3bc9acb..49db9808f 100644 --- a/lib/pleroma/web/plugs/uploaded_media.ex +++ b/lib/pleroma/web/plugs/uploaded_media.ex @@ -46,12 +46,21 @@ defmodule Pleroma.Web.Plugs.UploadedMedia do config = Pleroma.Config.get(Pleroma.Upload) - with uploader <- Keyword.fetch!(config, :uploader), + media_host = Pleroma.Upload.base_url() |> URI.parse() |> Map.get(:host) + + with {:valid_host, true} <- {:valid_host, match?(^media_host, conn.host)}, + uploader <- Keyword.fetch!(config, :uploader), proxy_remote = Keyword.get(config, :proxy_remote, false), {:ok, get_method} <- uploader.get_file(file), false <- media_is_banned(conn, get_method) do get_media(conn, get_method, proxy_remote, opts) else + {:valid_host, false} -> + conn + + send_resp(conn, 400, Plug.Conn.Status.reason_phrase(400)) + |> halt() + _ -> conn |> send_resp(:internal_server_error, dgettext("errors", "Failed")) diff --git a/test/pleroma/web/plugs/uploaded_media_plug_test.exs b/test/pleroma/web/plugs/uploaded_media_plug_test.exs index 8323ff6ab..262c8c288 100644 --- a/test/pleroma/web/plugs/uploaded_media_plug_test.exs +++ b/test/pleroma/web/plugs/uploaded_media_plug_test.exs @@ -40,4 +40,18 @@ defmodule Pleroma.Web.Plugs.UploadedMediaPlugTest do &(&1 == {"content-disposition", ~s[inline; filename="\\"cofe\\".gif"]}) ) end + + test "denies access to media if wrong Host", %{ + attachment_url: attachment_url + } do + conn = get(build_conn(), attachment_url) + + assert conn.status == 200 + + clear_config([Pleroma.Upload, :base_url], "http://media.localhost/") + + conn = get(build_conn(), attachment_url) + + assert conn.status == 400 + end end From 84974efe4c6e26f20bc14fece6cc74efe8777547 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 29 May 2023 14:17:27 -0400 Subject: [PATCH 76/89] Host header validation is now required for MediaProxy and Uploads --- changelog.d/3896.add | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3896.add diff --git a/changelog.d/3896.add b/changelog.d/3896.add new file mode 100644 index 000000000..3124e07dd --- /dev/null +++ b/changelog.d/3896.add @@ -0,0 +1 @@ +Validate Host header for MediaProxy and Uploads From 43bb2f39db0bd7028512cba402f6b5aa1610c1db Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 29 May 2023 15:05:37 -0400 Subject: [PATCH 77/89] Remove unwanted parameter --- test/support/conn_case.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/support/conn_case.ex b/test/support/conn_case.ex index d6729ab87..c1cb0295b 100644 --- a/test/support/conn_case.ex +++ b/test/support/conn_case.ex @@ -123,7 +123,6 @@ defmodule Pleroma.Web.ConnCase do {:ok, conn: Phoenix.ConnTest.build_conn() - |> Map.put(:host, Pleroma.Web.Endpoint.host()) - |> Plug.Test.init_test_session(%{})} + |> Map.put(:host, Pleroma.Web.Endpoint.host())} end end From da7394f33b4402325b220dcd2957945464517f8b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 29 May 2023 15:09:31 -0400 Subject: [PATCH 78/89] Fix unused assignment --- lib/pleroma/web/plugs/uploaded_media.ex | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/pleroma/web/plugs/uploaded_media.ex b/lib/pleroma/web/plugs/uploaded_media.ex index 49db9808f..f97c64cee 100644 --- a/lib/pleroma/web/plugs/uploaded_media.ex +++ b/lib/pleroma/web/plugs/uploaded_media.ex @@ -56,8 +56,6 @@ defmodule Pleroma.Web.Plugs.UploadedMedia do get_media(conn, get_method, proxy_remote, opts) else {:valid_host, false} -> - conn - send_resp(conn, 400, Plug.Conn.Status.reason_phrase(400)) |> halt() From 9caa0b0be1feffb20b3eaccb48298de572d32393 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 29 May 2023 15:46:49 -0400 Subject: [PATCH 79/89] Add OnlyMedia Upload Filter to simplify restricting uploads to audio, image, and video types --- changelog.d/3897.add | 1 + docs/configuration/cheatsheet.md | 6 ++++ lib/pleroma/upload/filter.ex | 6 ++-- lib/pleroma/upload/filter/only_media.ex | 20 ++++++++++++ .../pleroma/upload/filter/only_media_test.exs | 32 +++++++++++++++++++ 5 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 changelog.d/3897.add create mode 100644 lib/pleroma/upload/filter/only_media.ex create mode 100644 test/pleroma/upload/filter/only_media_test.exs diff --git a/changelog.d/3897.add b/changelog.d/3897.add new file mode 100644 index 000000000..5c4402f45 --- /dev/null +++ b/changelog.d/3897.add @@ -0,0 +1 @@ +OnlyMedia Upload Filter diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 70abe3e0c..026753f38 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -671,6 +671,12 @@ This filter reads the ImageDescription and iptc:Caption-Abstract fields with Exi No specific configuration. +#### Pleroma.Upload.Filter.OnlyMedia + +This filter rejects uploads that are not identified with Content-Type matching audio/*, image/*, or video/* + +No specific configuration. + #### Pleroma.Upload.Filter.Mogrify * `args`: List of actions for the `mogrify` command like `"strip"` or `["strip", "auto-orient", {"implode", "1"}]`. diff --git a/lib/pleroma/upload/filter.ex b/lib/pleroma/upload/filter.ex index 717f06621..809bc6e70 100644 --- a/lib/pleroma/upload/filter.ex +++ b/lib/pleroma/upload/filter.ex @@ -38,9 +38,9 @@ defmodule Pleroma.Upload.Filter do {:ok, :noop} -> filter(rest, upload) - error -> - Logger.error("#{__MODULE__}: Filter #{filter} failed: #{inspect(error)}") - error + {:error, e} -> + Logger.error("#{__MODULE__}: Filter #{filter} failed: #{inspect(e)}") + {:error, e} end end end diff --git a/lib/pleroma/upload/filter/only_media.ex b/lib/pleroma/upload/filter/only_media.ex new file mode 100644 index 000000000..a9caeba67 --- /dev/null +++ b/lib/pleroma/upload/filter/only_media.ex @@ -0,0 +1,20 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2023 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.Filter.OnlyMedia do + @behaviour Pleroma.Upload.Filter + alias Pleroma.Upload + + def filter(%Upload{content_type: content_type}) do + [type, _subtype] = String.split(content_type, "/") + + if type in ["image", "video", "audio"] do + {:ok, :noop} + else + {:error, "Disallowed content-type: #{content_type}"} + end + end + + def filter(_), do: {:ok, :noop} +end diff --git a/test/pleroma/upload/filter/only_media_test.exs b/test/pleroma/upload/filter/only_media_test.exs new file mode 100644 index 000000000..75be070a1 --- /dev/null +++ b/test/pleroma/upload/filter/only_media_test.exs @@ -0,0 +1,32 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2023 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.Filter.OnlyMediaTest do + use Pleroma.DataCase, async: true + + alias Pleroma.Upload + alias Pleroma.Upload.Filter.OnlyMedia + + test "Allows media Content-Type" do + ["audio/mpeg", "image/jpeg", "video/mp4"] + |> Enum.each(fn type -> + upload = %Upload{ + content_type: type + } + + assert {:ok, :noop} = OnlyMedia.filter(upload) + end) + end + + test "Disallows non-media Content-Type" do + ["application/javascript", "application/pdf", "text/html"] + |> Enum.each(fn type -> + upload = %Upload{ + content_type: type + } + + assert {:error, _} = OnlyMedia.filter(upload) + end) + end +end From 50a20f3bbd1203679c9295eb6002225dd630baf8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 29 May 2023 15:53:16 -0400 Subject: [PATCH 80/89] Esacpe the asterisks in Markdown --- docs/configuration/cheatsheet.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 026753f38..1e49a79d0 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -673,7 +673,7 @@ No specific configuration. #### Pleroma.Upload.Filter.OnlyMedia -This filter rejects uploads that are not identified with Content-Type matching audio/*, image/*, or video/* +This filter rejects uploads that are not identified with Content-Type matching audio/\*, image/\*, or video/\* No specific configuration. From b3c3bd99c390a4e5081d411011688e38285547b0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 30 May 2023 16:56:09 -0400 Subject: [PATCH 81/89] Switch from serving a 400 to a 302 --- changelog.d/3896.add | 2 +- .../web/media_proxy/media_proxy_controller.ex | 17 ++++++++++-- lib/pleroma/web/plugs/uploaded_media.ex | 17 ++++++++++-- .../media_proxy_controller_test.exs | 27 +++++++++++++++---- .../web/plugs/uploaded_media_plug_test.exs | 21 +++++++++++++-- 5 files changed, 72 insertions(+), 12 deletions(-) diff --git a/changelog.d/3896.add b/changelog.d/3896.add index 3124e07dd..36d8286ff 100644 --- a/changelog.d/3896.add +++ b/changelog.d/3896.add @@ -1 +1 @@ -Validate Host header for MediaProxy and Uploads +Validate Host header for MediaProxy and Uploads and return a 302 if the base_url has changed diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index 767496b68..20f3a3438 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -207,12 +207,25 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do end defp validate_host(conn, _params) do - proxy_host = MediaProxy.base_url() |> URI.parse() |> Map.get(:host) + %{scheme: proxy_scheme, host: proxy_host, port: proxy_port} = + MediaProxy.base_url() |> URI.parse() if match?(^proxy_host, conn.host) do conn else - send_resp(conn, 400, Conn.Status.reason_phrase(400)) + redirect_url = + %URI{ + scheme: proxy_scheme, + host: proxy_host, + port: proxy_port, + path: conn.request_path, + query: conn.query_string + } + |> URI.to_string() + |> String.trim_trailing("?") + + conn + |> Phoenix.Controller.redirect(external: redirect_url) |> halt() end end diff --git a/lib/pleroma/web/plugs/uploaded_media.ex b/lib/pleroma/web/plugs/uploaded_media.ex index f97c64cee..9dd5eb239 100644 --- a/lib/pleroma/web/plugs/uploaded_media.ex +++ b/lib/pleroma/web/plugs/uploaded_media.ex @@ -46,7 +46,8 @@ defmodule Pleroma.Web.Plugs.UploadedMedia do config = Pleroma.Config.get(Pleroma.Upload) - media_host = Pleroma.Upload.base_url() |> URI.parse() |> Map.get(:host) + %{scheme: media_scheme, host: media_host, port: media_port} = + Pleroma.Upload.base_url() |> URI.parse() with {:valid_host, true} <- {:valid_host, match?(^media_host, conn.host)}, uploader <- Keyword.fetch!(config, :uploader), @@ -56,7 +57,19 @@ defmodule Pleroma.Web.Plugs.UploadedMedia do get_media(conn, get_method, proxy_remote, opts) else {:valid_host, false} -> - send_resp(conn, 400, Plug.Conn.Status.reason_phrase(400)) + redirect_url = + %URI{ + scheme: media_scheme, + host: media_host, + port: media_port, + path: conn.request_path, + query: conn.query_string + } + |> URI.to_string() + |> String.trim_trailing("?") + + conn + |> Phoenix.Controller.redirect(external: redirect_url) |> halt() _ -> diff --git a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs index 3971330d4..019e389b7 100644 --- a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs +++ b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs @@ -54,20 +54,37 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do } = get(conn, "/proxy/hhgfh/eeee/fff") end - test "it returns a 400 for invalid host", %{conn: conn} do - clear_config([:media_proxy, :base_url], "http://mp.localhost/") + test "it returns a 302 for invalid host", %{conn: conn} do + new_proxy_base = "http://mp.localhost/" - url = + %{scheme: new_proxy_scheme, host: new_proxy_host, port: new_proxy_port} = + URI.parse(new_proxy_base) + + clear_config([:media_proxy, :base_url], new_proxy_base) + + proxy_url = MediaProxy.encode_url("https://pleroma.social/logo.jpeg") |> URI.parse() |> Map.put(:host, "wronghost") |> URI.to_string() + expected_url = + URI.parse(proxy_url) + |> Map.put(:host, new_proxy_host) + |> Map.put(:port, new_proxy_port) + |> Map.put(:scheme, new_proxy_scheme) + |> URI.to_string() + with_mock Pleroma.ReverseProxy, call: fn _conn, _url, _opts -> %Conn{status: :success} end do - %{status: status} = get(conn, url) + %{resp_headers: resp_headers, status: status} = get(conn, proxy_url) - assert status == 400 + assert status == 302 + + assert Enum.any?( + resp_headers, + &(&1 == {"location", expected_url}) + ) end end diff --git a/test/pleroma/web/plugs/uploaded_media_plug_test.exs b/test/pleroma/web/plugs/uploaded_media_plug_test.exs index 262c8c288..fcc523cf4 100644 --- a/test/pleroma/web/plugs/uploaded_media_plug_test.exs +++ b/test/pleroma/web/plugs/uploaded_media_plug_test.exs @@ -48,10 +48,27 @@ defmodule Pleroma.Web.Plugs.UploadedMediaPlugTest do assert conn.status == 200 - clear_config([Pleroma.Upload, :base_url], "http://media.localhost/") + new_media_base = "http://media.localhost:8080" + + %{scheme: new_media_scheme, host: new_media_host, port: new_media_port} = + URI.parse(new_media_base) + + clear_config([Pleroma.Upload, :base_url], new_media_base) conn = get(build_conn(), attachment_url) - assert conn.status == 400 + expected_url = + URI.parse(attachment_url) + |> Map.put(:host, new_media_host) + |> Map.put(:port, new_media_port) + |> Map.put(:scheme, new_media_scheme) + |> URI.to_string() + + assert conn.status == 302 + + assert Enum.any?( + conn.resp_headers, + &(&1 == {"location", expected_url}) + ) end end From 8336519f300e2e12087126564a155c292915daed Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 31 May 2023 08:26:21 +0200 Subject: [PATCH 82/89] installation/debian_based_en: Elixir 1.11 means Debian 12+ and Ubuntu 22.04+ --- docs/installation/debian_based_en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md index 4e52b2155..575b8a7ab 100644 --- a/docs/installation/debian_based_en.md +++ b/docs/installation/debian_based_en.md @@ -4,7 +4,7 @@ ## Installation -This guide will assume you are on Debian 11 (“bullseye”) or later. This guide should also work with Ubuntu 18.04 (“Bionic Beaver”) and later. It also assumes that you have administrative rights, either as root or a user with [sudo permissions](https://www.digitalocean.com/community/tutorials/how-to-add-delete-and-grant-sudo-privileges-to-users-on-a-debian-vps). If you want to run this guide with root, ignore the `sudo` at the beginning of the lines, unless it calls a user like `sudo -Hu pleroma`; in this case, use `su -s $SHELL -c 'command'` instead. +This guide will assume you are on Debian 12 (“bookworm”) or later. This guide should also work with Ubuntu 22.04 (“jammy”) and later. It also assumes that you have administrative rights, either as root or a user with [sudo permissions](https://www.digitalocean.com/community/tutorials/how-to-add-delete-and-grant-sudo-privileges-to-users-on-a-debian-vps). If you want to run this guide with root, ignore the `sudo` at the beginning of the lines, unless it calls a user like `sudo -Hu pleroma`; in this case, use `su -s $SHELL -c 'command'` instead. {! backend/installation/generic_dependencies.include !} From 737e45c1022172b316671110468a03a22a23e809 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 31 May 2023 08:28:28 +0200 Subject: [PATCH 83/89] installation/debian_based_jp: Elixir 1.11 means Debian 12+ and Ubuntu 22.04+ I checked for what each part of the debian&ubuntu sentences meant with the help of Jisho.org, should be safe but it did make me notice that this guide hasn't been updated in years --- changelog.d/distro-docs-elixir-1.11.skip | 0 docs/installation/debian_based_jp.md | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 changelog.d/distro-docs-elixir-1.11.skip diff --git a/changelog.d/distro-docs-elixir-1.11.skip b/changelog.d/distro-docs-elixir-1.11.skip new file mode 100644 index 000000000..e69de29bb diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md index 3736e857f..695e7bb7d 100644 --- a/docs/installation/debian_based_jp.md +++ b/docs/installation/debian_based_jp.md @@ -5,7 +5,7 @@ ## インストール -このガイドはDebian Stretchを利用することを想定しています。Ubuntu 16.04や18.04でもおそらく動作します。また、ユーザはrootもしくはsudoにより管理者権限を持っていることを前提とします。もし、以下の操作をrootユーザで行う場合は、 `sudo` を無視してください。ただし、`sudo -Hu pleroma` のようにユーザを指定している場合には `su -s $SHELL -c 'command'` を代わりに使ってください。 +このガイドはDebian Bookwormを利用することを想定しています。Ubuntu 22.04でもおそらく動作します。また、ユーザはrootもしくはsudoにより管理者権限を持っていることを前提とします。もし、以下の操作をrootユーザで行う場合は、 `sudo` を無視してください。ただし、`sudo -Hu pleroma` のようにユーザを指定している場合には `su -s $SHELL -c 'command'` を代わりに使ってください。 ### 必要なソフトウェア From 46c799f5284afe26756a6dd1b247af32de3be929 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 May 2023 09:50:47 -0400 Subject: [PATCH 84/89] Use Phoenix.ConnTest.redirected_to/2 --- changelog.d/3899.skip | 0 .../web/media_proxy/media_proxy_controller_test.exs | 9 ++------- test/pleroma/web/plugs/uploaded_media_plug_test.exs | 7 +------ 3 files changed, 3 insertions(+), 13 deletions(-) create mode 100644 changelog.d/3899.skip diff --git a/changelog.d/3899.skip b/changelog.d/3899.skip new file mode 100644 index 000000000..e69de29bb diff --git a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs index 019e389b7..deb407709 100644 --- a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs +++ b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs @@ -77,14 +77,9 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do with_mock Pleroma.ReverseProxy, call: fn _conn, _url, _opts -> %Conn{status: :success} end do - %{resp_headers: resp_headers, status: status} = get(conn, proxy_url) + conn = get(conn, proxy_url) - assert status == 302 - - assert Enum.any?( - resp_headers, - &(&1 == {"location", expected_url}) - ) + assert redirected_to(conn, 302) == expected_url end end diff --git a/test/pleroma/web/plugs/uploaded_media_plug_test.exs b/test/pleroma/web/plugs/uploaded_media_plug_test.exs index fcc523cf4..dbf8ca5ec 100644 --- a/test/pleroma/web/plugs/uploaded_media_plug_test.exs +++ b/test/pleroma/web/plugs/uploaded_media_plug_test.exs @@ -64,11 +64,6 @@ defmodule Pleroma.Web.Plugs.UploadedMediaPlugTest do |> Map.put(:scheme, new_media_scheme) |> URI.to_string() - assert conn.status == 302 - - assert Enum.any?( - conn.resp_headers, - &(&1 == {"location", expected_url}) - ) + assert redirected_to(conn, 302) == expected_url end end From 313e68c1809a281a781b221778c3e3c44728cca0 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 6 Jan 2023 21:53:17 +0100 Subject: [PATCH 85/89] mix: bump gettext to ~0.20 Includes https://github.com/elixir-gettext/gettext/pull/304 in 0.20.0+ Includes https://github.com/elixir-gettext/expo/issues/91 in 0.22+ via expo 0.2.0+ --- changelog.d/3831.skip | 0 mix.exs | 7 ++----- mix.lock | 3 ++- priv/gettext/en_test/LC_MESSAGES/default.po | 1 - priv/gettext/en_test/LC_MESSAGES/errors.po | 1 - priv/gettext/en_test/LC_MESSAGES/posix_errors.po | 1 - priv/gettext/en_test/LC_MESSAGES/static_pages.po | 4 ---- priv/gettext/ru/LC_MESSAGES/errors.po | 1 - priv/gettext/zh_Hans/LC_MESSAGES/static_pages.po | 4 ---- 9 files changed, 4 insertions(+), 18 deletions(-) create mode 100644 changelog.d/3831.skip diff --git a/changelog.d/3831.skip b/changelog.d/3831.skip new file mode 100644 index 000000000..e69de29bb diff --git a/mix.exs b/mix.exs index bb16f0a9c..62ac1c1a0 100644 --- a/mix.exs +++ b/mix.exs @@ -7,7 +7,7 @@ defmodule Pleroma.Mixfile do version: version("2.5.52"), elixir: "~> 1.11", elixirc_paths: elixirc_paths(Mix.env()), - compilers: [:phoenix, :gettext] ++ Mix.compilers(), + compilers: [:phoenix] ++ Mix.compilers(), elixirc_options: [warnings_as_errors: warnings_as_errors()], xref: [exclude: [:eldap]], start_permanent: Mix.env() == :prod, @@ -127,10 +127,7 @@ defmodule Pleroma.Mixfile do {:plug_cowboy, "~> 2.3"}, # oban 2.14 requires Elixir 1.12+ {:oban, "~> 2.13.4"}, - {:gettext, - git: "https://github.com/tusooa/gettext.git", - ref: "72fb2496b6c5280ed911bdc3756890e7f38a4808", - override: true}, + {:gettext, "~> 0.20"}, {:bcrypt_elixir, "~> 2.2"}, {:trailing_format_plug, "~> 0.0.7"}, {:fast_sanitize, "~> 0.2.0"}, diff --git a/mix.lock b/mix.lock index 43012a58e..b6938f44b 100644 --- a/mix.lock +++ b/mix.lock @@ -42,6 +42,7 @@ "ex_doc": {:hex, :ex_doc, "0.29.4", "6257ecbb20c7396b1fe5accd55b7b0d23f44b6aa18017b415cb4c2b91d997729", [:mix], [{:earmark_parser, "~> 1.4.31", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "2c6699a737ae46cb61e4ed012af931b57b699643b24dabe2400a8168414bc4f5"}, "ex_machina": {:hex, :ex_machina, "2.7.0", "b792cc3127fd0680fecdb6299235b4727a4944a09ff0fa904cc639272cd92dc7", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "419aa7a39bde11894c87a615c4ecaa52d8f107bbdd81d810465186f783245bf8"}, "ex_syslogger": {:hex, :ex_syslogger, "1.5.2", "72b6aa2d47a236e999171f2e1ec18698740f40af0bd02c8c650bf5f1fd1bac79", [:mix], [{:poison, ">= 1.5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:syslog, "~> 1.1.0", [hex: :syslog, repo: "hexpm", optional: false]}], "hexpm", "ab9fab4136dbc62651ec6f16fa4842f10cf02ab4433fa3d0976c01be99398399"}, + "expo": {:hex, :expo, "0.4.1", "1c61d18a5df197dfda38861673d392e642649a9cef7694d2f97a587b2cfb319b", [:mix], [], "hexpm", "2ff7ba7a798c8c543c12550fa0e2cbc81b95d4974c65855d8d15ba7b37a1ce47"}, "fast_html": {:hex, :fast_html, "2.0.5", "c61760340606c1077ff1f196f17834056cb1dd3d5cb92a9f2cabf28bc6221c3c", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}], "hexpm", "605f4f4829443c14127694ebabb681778712ceecb4470ec32aa31012330e6506"}, "fast_sanitize": {:hex, :fast_sanitize, "0.2.3", "67b93dfb34e302bef49fec3aaab74951e0f0602fd9fa99085987af05bd91c7a5", [:mix], [{:fast_html, "~> 2.0", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "e8ad286d10d0386e15d67d0ee125245ebcfbc7d7290b08712ba9013c8c5e56e2"}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, @@ -49,7 +50,7 @@ "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "31fc8090fde1acd267c07c36ea7365b8604055f897d3a53dd967658c691bd827"}, "floki": {:hex, :floki, "0.34.2", "5fad07ef153b3b8ec110b6b155ec3780c4b2c4906297d0b4be1a7162d04a7e02", [:mix], [], "hexpm", "26b9d50f0f01796bc6be611ca815c5e0de034d2128e39cc9702eee6b66a4d1c8"}, "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm", "29bd14a88030980849c7ed2447b8db6d6c9278a28b11a44cafe41b791205440f"}, - "gettext": {:git, "https://github.com/tusooa/gettext.git", "72fb2496b6c5280ed911bdc3756890e7f38a4808", [ref: "72fb2496b6c5280ed911bdc3756890e7f38a4808"]}, + "gettext": {:hex, :gettext, "0.22.2", "6bfca374de34ecc913a28ba391ca184d88d77810a3e427afa8454a71a51341ac", [:mix], [{:expo, "~> 0.4.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "8a2d389673aea82d7eae387e6a2ccc12660610080ae7beb19452cfdc1ec30f60"}, "gun": {:hex, :gun, "2.0.0", "2326bc0fd6d9cf628419708270d6fe8b02b8d002cf992e4165a77d997b1defd0", [:make, :rebar3], [{:cowlib, "2.12.0", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm", "6613cb7c62930dc8d58263c44dda72f8556346ba88358fc929dcbc5f76d04569"}, "hackney": {:hex, :hackney, "1.18.1", "f48bf88f521f2a229fc7bae88cf4f85adc9cd9bcf23b5dc8eb6a1788c662c4f6", [:rebar3], [{:certifi, "~>2.9.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a4ecdaff44297e9b5894ae499e9a070ea1888c84afdd1fd9b7b2bc384950128e"}, "hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"}, diff --git a/priv/gettext/en_test/LC_MESSAGES/default.po b/priv/gettext/en_test/LC_MESSAGES/default.po index 63db74608..037e14466 100644 --- a/priv/gettext/en_test/LC_MESSAGES/default.po +++ b/priv/gettext/en_test/LC_MESSAGES/default.po @@ -9,7 +9,6 @@ msgid "" msgstr "" "Language: en_test\n" -"Plural-Forms: nplurals=2\n" #, elixir-format #: lib/pleroma/web/api_spec/render_error.ex:122 diff --git a/priv/gettext/en_test/LC_MESSAGES/errors.po b/priv/gettext/en_test/LC_MESSAGES/errors.po index a40de7f8b..286bbb1aa 100644 --- a/priv/gettext/en_test/LC_MESSAGES/errors.po +++ b/priv/gettext/en_test/LC_MESSAGES/errors.po @@ -9,7 +9,6 @@ msgid "" msgstr "" "Language: en_test\n" -"Plural-Forms: nplurals=2\n" msgid "can't be blank" msgstr "" diff --git a/priv/gettext/en_test/LC_MESSAGES/posix_errors.po b/priv/gettext/en_test/LC_MESSAGES/posix_errors.po index 663fc5924..6ff9dc53d 100644 --- a/priv/gettext/en_test/LC_MESSAGES/posix_errors.po +++ b/priv/gettext/en_test/LC_MESSAGES/posix_errors.po @@ -9,7 +9,6 @@ msgid "" msgstr "" "Language: en_test\n" -"Plural-Forms: nplurals=2\n" msgid "eperm" msgstr "" diff --git a/priv/gettext/en_test/LC_MESSAGES/static_pages.po b/priv/gettext/en_test/LC_MESSAGES/static_pages.po index 1a3b7b355..daf312093 100644 --- a/priv/gettext/en_test/LC_MESSAGES/static_pages.po +++ b/priv/gettext/en_test/LC_MESSAGES/static_pages.po @@ -21,10 +21,6 @@ msgstr "" #~ ## #~ ## Use "mix gettext.extract --merge" or "mix gettext.merge" #~ ## to merge POT files into PO files. -#~ msgid "" -#~ msgstr "" -#~ "Language: en_test\n" -#~ "Plural-Forms: nplurals=2\n" #, elixir-format #: lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex:9 diff --git a/priv/gettext/ru/LC_MESSAGES/errors.po b/priv/gettext/ru/LC_MESSAGES/errors.po index 39f83e8a6..64218da6f 100644 --- a/priv/gettext/ru/LC_MESSAGES/errors.po +++ b/priv/gettext/ru/LC_MESSAGES/errors.po @@ -9,7 +9,6 @@ msgid "" msgstr "" "Language: ru\n" -"Plural-Forms: nplurals=3\n" msgid "can't be blank" msgstr "не может быть пустым" diff --git a/priv/gettext/zh_Hans/LC_MESSAGES/static_pages.po b/priv/gettext/zh_Hans/LC_MESSAGES/static_pages.po index cbd6feb60..809b13d47 100644 --- a/priv/gettext/zh_Hans/LC_MESSAGES/static_pages.po +++ b/priv/gettext/zh_Hans/LC_MESSAGES/static_pages.po @@ -24,10 +24,6 @@ msgstr "" ## ## Use "mix gettext.extract --merge" or "mix gettext.merge" ## to merge POT files into PO files. -#~ msgid "" -#~ msgstr "" -#~ "Language: zh_Hans\n" -#~ "Plural-Forms: nplurals=1\n" #: lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex:9 #, elixir-format From cbc5b8cebd9255e0c49e8fb02daed4680be1d336 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 2 Jun 2023 17:03:21 +0400 Subject: [PATCH 86/89] B Preload: Make sure that the preloaded json is html safe --- lib/pleroma/web/preload.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/preload.ex b/lib/pleroma/web/preload.ex index 4485383f9..6a4a8885e 100644 --- a/lib/pleroma/web/preload.ex +++ b/lib/pleroma/web/preload.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Web.Preload do terms = params |> parser.generate_terms() - |> Enum.map(fn {k, v} -> {k, Base.encode64(Jason.encode!(v))} end) + |> Enum.map(fn {k, v} -> {k, Base.encode64(Jason.encode!(v, escape: :html_safe))} end) |> Enum.into(%{}) Map.merge(acc, terms) @@ -19,7 +19,7 @@ defmodule Pleroma.Web.Preload do rendered_html = preload_data - |> Jason.encode!() + |> Jason.encode!(escape: :html_safe) |> build_script_tag() |> HTML.safe_to_string() From 40d40d67a3cee4d57f9200d0980df1b21d08a834 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Fri, 2 Jun 2023 17:09:23 +0400 Subject: [PATCH 87/89] Add changelog. --- changelog.d/3901.security | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3901.security diff --git a/changelog.d/3901.security b/changelog.d/3901.security new file mode 100644 index 000000000..a3d8bd01f --- /dev/null +++ b/changelog.d/3901.security @@ -0,0 +1 @@ +Preload: Make generated JSON html-safe. It already was html safe because it only consists of config data that is base64 encoded, but this will keep it safe it that ever changes. From fadcd7f1a9e41ec9b54f251cf782688bf3d36889 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Jun 2023 09:19:22 -0400 Subject: [PATCH 88/89] Revert MediaProxy Host header validation Something is going wrong here even though the tests are correct. --- changelog.d/3896.add | 2 +- .../web/media_proxy/media_proxy_controller.ex | 25 ---------------- .../media_proxy_controller_test.exs | 29 ------------------- test/support/conn_case.ex | 5 +--- 4 files changed, 2 insertions(+), 59 deletions(-) diff --git a/changelog.d/3896.add b/changelog.d/3896.add index 36d8286ff..e8fde4c8a 100644 --- a/changelog.d/3896.add +++ b/changelog.d/3896.add @@ -1 +1 @@ -Validate Host header for MediaProxy and Uploads and return a 302 if the base_url has changed +Validate Host header for Uploads and return a 302 if the base_url has changed diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index 20f3a3438..bda5b36ed 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -12,7 +12,6 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do alias Pleroma.Web.MediaProxy alias Plug.Conn - plug(:validate_host) plug(:sandbox) def remote(conn, %{"sig" => sig64, "url" => url64}) do @@ -206,30 +205,6 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do Config.get([:media_proxy, :proxy_opts], []) end - defp validate_host(conn, _params) do - %{scheme: proxy_scheme, host: proxy_host, port: proxy_port} = - MediaProxy.base_url() |> URI.parse() - - if match?(^proxy_host, conn.host) do - conn - else - redirect_url = - %URI{ - scheme: proxy_scheme, - host: proxy_host, - port: proxy_port, - path: conn.request_path, - query: conn.query_string - } - |> URI.to_string() - |> String.trim_trailing("?") - - conn - |> Phoenix.Controller.redirect(external: redirect_url) - |> halt() - end - end - defp sandbox(conn, _params) do conn |> merge_resp_headers([{"content-security-policy", "sandbox;"}]) diff --git a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs index deb407709..9ce092fd8 100644 --- a/test/pleroma/web/media_proxy/media_proxy_controller_test.exs +++ b/test/pleroma/web/media_proxy/media_proxy_controller_test.exs @@ -54,35 +54,6 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do } = get(conn, "/proxy/hhgfh/eeee/fff") end - test "it returns a 302 for invalid host", %{conn: conn} do - new_proxy_base = "http://mp.localhost/" - - %{scheme: new_proxy_scheme, host: new_proxy_host, port: new_proxy_port} = - URI.parse(new_proxy_base) - - clear_config([:media_proxy, :base_url], new_proxy_base) - - proxy_url = - MediaProxy.encode_url("https://pleroma.social/logo.jpeg") - |> URI.parse() - |> Map.put(:host, "wronghost") - |> URI.to_string() - - expected_url = - URI.parse(proxy_url) - |> Map.put(:host, new_proxy_host) - |> Map.put(:port, new_proxy_port) - |> Map.put(:scheme, new_proxy_scheme) - |> URI.to_string() - - with_mock Pleroma.ReverseProxy, - call: fn _conn, _url, _opts -> %Conn{status: :success} end do - conn = get(conn, proxy_url) - - assert redirected_to(conn, 302) == expected_url - end - end - test "redirects to valid url when filename is invalidated", %{conn: conn, url: url} do invalid_url = String.replace(url, "test.png", "test-file.png") response = get(conn, invalid_url) diff --git a/test/support/conn_case.ex b/test/support/conn_case.ex index c1cb0295b..f010fec33 100644 --- a/test/support/conn_case.ex +++ b/test/support/conn_case.ex @@ -120,9 +120,6 @@ defmodule Pleroma.Web.ConnCase do Mox.verify_on_exit!() - {:ok, - conn: - Phoenix.ConnTest.build_conn() - |> Map.put(:host, Pleroma.Web.Endpoint.host())} + {:ok, conn: Phoenix.ConnTest.build_conn()} end end From 1ca1b4b32f2cba5f78a2d55645b8d2814bbfc593 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Jun 2023 09:25:57 -0400 Subject: [PATCH 89/89] changelog.d --- changelog.d/3902.skip | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 changelog.d/3902.skip diff --git a/changelog.d/3902.skip b/changelog.d/3902.skip new file mode 100644 index 000000000..e69de29bb