Always insert Dedupe upload filter

This actually was already intended before to eradict all future
path-traversal-style exploits and to fix issues with some
characters like akkoma#610 in 0b2ec0ccee. However, Dedupe and
AnonymizeFilename got mixed up. The latter only anonymises the name
in Content-Disposition headers GET parameters (with link_name),
_not_ the upload path.

Even without Dedupe, the upload path is prefixed by an UUID,
so it _should_ already be hard to guess for attackers. But now
we actually can be sure no path shenanigangs occur, uploads
reliably work and save some disk space.

While this makes the final path predictable, this prediction is
not exploitable. Insertion of a back-reference to the upload
itself requires pulling off a successfull preimage attack against
SHA-256, which is deemed infeasible for the foreseeable futures.

Dedupe was already included in the default list in config.exs
since 28cfb2c37a, but this will get overridde by whatever the
config generated by the "pleroma.instance gen" task chose.

Upload+delete tests running in parallel using Dedupe might be flaky, but
this was already true before and needs its own commit to fix eventually.

CHANGELOG.md

@@ -11,6 +11,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## Changed
 - `Pleroma.Upload, :base_url` now MUST be configured explicitly;
   use of the same domain as the instance is **strongly** discouraged
+- The `Dedupe` upload filter is now always active;
+  `AnonymizeFilenames` is again opt-in
 
 ## Fixed
 - Critical security issue allowing Akkoma to be used as a vector for

config/config.exs

@@ -61,7 +61,7 @@ config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.
 # Upload configuration
 config :pleroma, Pleroma.Upload,
   uploader: Pleroma.Uploaders.Local,
-  filters: [Pleroma.Upload.Filter.Dedupe],
+  filters: [],
   link_name: false,
   proxy_remote: false,
   filename_display_max_length: 30,

docs/docs/configuration/cheatsheet.md

@@ -597,7 +597,7 @@ the source code is here: [kocaptcha](https://github.com/koto-bank/kocaptcha). Th
 
 * `uploader`: Which one of the [uploaders](#uploaders) to use.
 * `filters`: List of [upload filters](#upload-filters) to use.
-* `link_name`: When enabled Akkoma will add a `name` parameter to the url of the upload, for example `https://instance.tld/media/corndog.png?name=corndog.png`. This is needed to provide the correct filename in Content-Disposition headers when using filters like `Pleroma.Upload.Filter.Dedupe`
+* `link_name`: When enabled Akkoma will add a `name` parameter to the url of the upload, for example `https://instance.tld/media/corndog.png?name=corndog.png`. This is needed to provide the correct filename in Content-Disposition headers
 * `base_url`: The base URL to access a user-uploaded file; MUST be configured explicitly.
   Using a (sub)domain distinct from the instance endpoint is **strongly** recommended.
 * `proxy_remote`: If you're using a remote uploader, Akkoma will proxy media requests instead of redirecting to it.
@@ -639,17 +639,18 @@ config :ex_aws, :s3,
 
 ### Upload filters
 
-#### Pleroma.Upload.Filter.AnonymizeFilename
-
-This filter replaces the filename (not the path) of an upload. For complete obfuscation, add
-`Pleroma.Upload.Filter.Dedupe` before AnonymizeFilename.
-
-* `text`: Text to replace filenames in links. If empty, `{random}.extension` will be used. You can get the original filename extension by using `{extension}`, for example `custom-file-name.{extension}`.
-
 #### Pleroma.Upload.Filter.Dedupe
 
+**Always** active; cannot be turned off.
+Renames files to their hash and prevents duplicate files filling up the disk.
 No specific configuration.
 
+#### Pleroma.Upload.Filter.AnonymizeFilename
+
+This filter replaces the declared filename (not the path) of an upload.
+
+* `text`: Text to replace filenames in links. If empty, `{random}.extension` will be used. You can get the original filename extension by using `{extension}`, for example `custom-file-name.{extension}`.
+
 #### Pleroma.Upload.Filter.Exiftool
 
 This filter only strips the GPS and location metadata with Exiftool leaving color profiles and attributes intact.

lib/mix/tasks/pleroma/instance.ex

@@ -36,8 +36,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
           listen_ip: :string,
           listen_port: :string,
           strip_uploads: :string,
-          anonymize_uploads: :string,
+          anonymize_uploads: :string
-          dedupe_uploads: :string
         ],
         aliases: [
           o: :output,
@@ -195,14 +194,6 @@ defmodule Mix.Tasks.Pleroma.Instance do
           "n"
         ) === "y"
 
-      dedupe_uploads =
-        get_option(
-          options,
-          :dedupe_uploads,
-          "Do you want to deduplicate uploaded files? (y/n)",
-          "n"
-        ) === "y"
-
       Config.put([:instance, :static_dir], static_dir)
 
       secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64)
@@ -240,8 +231,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
           upload_filters:
             upload_filters(%{
               strip: strip_uploads,
-              anonymize: anonymize_uploads,
+              anonymize: anonymize_uploads
-              dedupe: dedupe_uploads
             })
         )
 
@@ -329,13 +319,6 @@ defmodule Mix.Tasks.Pleroma.Instance do
         enabled_filters
       end
 
-    enabled_filters =
-      if filters.dedupe do
-        enabled_filters ++ [Pleroma.Upload.Filter.Dedupe]
-      else
-        enabled_filters
-      end
-
     enabled_filters
   end
 end

lib/pleroma/upload.ex

@@ -66,7 +66,7 @@ defmodule Pleroma.Upload do
           path: String.t()
         }
 
-  @always_enabled_filters [Pleroma.Upload.Filter.AnonymizeFilename]
+  @always_enabled_filters [Pleroma.Upload.Filter.Dedupe]
 
   defstruct [:id, :name, :tempfile, :content_type, :width, :height, :blurhash, :path]
 

test/mix/tasks/pleroma/instance_test.exs

@@ -71,8 +71,6 @@ defmodule Mix.Tasks.Pleroma.InstanceTest do
         "./test/../test/instance/static/",
         "--strip-uploads",
         "y",
-        "--dedupe-uploads",
-        "n",
         "--anonymize-uploads",
         "n"
       ])

test/mix/tasks/pleroma/uploads_test.exs

@@ -16,7 +16,6 @@ defmodule Mix.Tasks.Pleroma.UploadsTest do
       Mix.shell(Mix.Shell.IO)
     end)
 
-    File.mkdir_p!("test/uploads")
     :ok
   end
 

test/pleroma/object_test.exs

@@ -22,6 +22,13 @@ defmodule Pleroma.ObjectTest do
     :ok
   end
 
+  # Only works for a single attachment but that's all we need here
+  defp get_attachment_filepath(note, uploads_dir) do
+    %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} = note
+    filename = href |> Path.basename()
+    "#{uploads_dir}/#{filename}"
+  end
+
   test "returns an object by it's AP id" do
     object = insert(:note)
     found_object = Object.get_by_ap_id(object.data["id"])
@@ -95,14 +102,13 @@ defmodule Pleroma.ObjectTest do
       {:ok, %Object{} = attachment} =
         Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)
 
-      %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
       note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})
 
       uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])
 
-      path = href |> Path.dirname() |> Path.basename()
+      path = get_attachment_filepath(note, uploads_dir)
 
-      assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")
+      assert File.exists?("#{path}")
 
       Object.delete(note)
 
@@ -111,7 +117,7 @@ defmodule Pleroma.ObjectTest do
       assert Object.get_by_id(note.id).data["deleted"]
       refute Object.get_by_id(attachment.id) == nil
 
-      assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")
+      assert File.exists?("#{path}")
     end
 
     test "in subdirectories" do
@@ -129,14 +135,13 @@ defmodule Pleroma.ObjectTest do
       {:ok, %Object{} = attachment} =
         Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)
 
-      %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
       note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})
 
       uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])
 
-      path = href |> Path.dirname() |> Path.basename()
+      path = get_attachment_filepath(note, uploads_dir)
 
-      assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")
+      assert File.exists?("#{path}")
 
       Object.delete(note)
 
@@ -145,7 +150,7 @@ defmodule Pleroma.ObjectTest do
       assert Object.get_by_id(note.id).data["deleted"]
       assert Object.get_by_id(attachment.id) == nil
 
-      assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")
+      refute File.exists?("#{path}")
     end
 
     test "with dedupe enabled" do
@@ -168,13 +173,11 @@ defmodule Pleroma.ObjectTest do
       {:ok, %Object{} = attachment} =
         Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)
 
-      %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
       note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})
 
-      filename = Path.basename(href)
+      path = get_attachment_filepath(note, uploads_dir)
 
-      assert {:ok, files} = File.ls(uploads_dir)
+      assert File.exists?("#{path}")
-      assert filename in files
 
       Object.delete(note)
 
@@ -182,8 +185,8 @@ defmodule Pleroma.ObjectTest do
 
       assert Object.get_by_id(note.id).data["deleted"]
       assert Object.get_by_id(attachment.id) == nil
-      assert {:ok, files} = File.ls(uploads_dir)
+      # what if another test runs concurrently using the same image file?
-      refute filename in files
+      refute File.exists?("#{path}")
     end
 
     test "with objects that have legacy data.url attribute" do
@@ -203,14 +206,13 @@ defmodule Pleroma.ObjectTest do
 
       {:ok, %Object{}} = Object.create(%{url: "https://google.com", actor: user.ap_id})
 
-      %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
       note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})
 
       uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])
 
-      path = href |> Path.dirname() |> Path.basename()
+      path = get_attachment_filepath(note, uploads_dir)
 
-      assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")
+      assert File.exists?("#{path}")
 
       Object.delete(note)
 
@@ -219,7 +221,7 @@ defmodule Pleroma.ObjectTest do
       assert Object.get_by_id(note.id).data["deleted"]
       assert Object.get_by_id(attachment.id) == nil
 
-      assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")
+      refute File.exists?("#{path}")
     end
 
     test "With custom base_url" do
@@ -238,14 +240,13 @@ defmodule Pleroma.ObjectTest do
       {:ok, %Object{} = attachment} =
         Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)
 
-      %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
       note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})
 
       uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])
 
-      path = href |> Path.dirname() |> Path.basename()
+      path = get_attachment_filepath(note, uploads_dir)
 
-      assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")
+      assert File.exists?("#{path}")
 
       Object.delete(note)
 
@@ -254,7 +255,7 @@ defmodule Pleroma.ObjectTest do
       assert Object.get_by_id(note.id).data["deleted"]
       assert Object.get_by_id(attachment.id) == nil
 
-      assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")
+      refute File.exists?("#{path}")
     end
   end
 

test/pleroma/upload_test.exs

@@ -188,7 +188,7 @@ defmodule Pleroma.UploadTest do
       refute data["name"] == "an [image.jpg"
     end
 
-    test "escapes invalid characters in url" do
+    test "mangles the filename" do
       File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
 
       file = %Plug.Upload{
@@ -200,23 +200,8 @@ defmodule Pleroma.UploadTest do
       {:ok, data} = Upload.store(file)
       [attachment_url | _] = data["url"]
 
-      assert Path.basename(attachment_url["href"]) == "an%E2%80%A6%20image.jpg"
+      refute Path.basename(attachment_url["href"]) == "an%E2%80%A6%20image.jpg"
-    end
+      refute Path.basename(attachment_url["href"]) == "an… image.jpg"
-
-    test "escapes reserved uri characters" do
-      File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
-
-      file = %Plug.Upload{
-        content_type: "image/jpeg",
-        path: Path.absname("test/fixtures/image_tmp.jpg"),
-        filename: ":?#[]@!$&\\'()*+,;=.jpg"
-      }
-
-      {:ok, data} = Upload.store(file)
-      [attachment_url | _] = data["url"]
-
-      assert Path.basename(attachment_url["href"]) ==
-               "%3A%3F%23%5B%5D%40%21%24%26%5C%27%28%29%2A%2B%2C%3B%3D.jpg"
     end
   end
 

test/pleroma/web/activity_pub/activity_pub_test.exs

@@ -1304,14 +1304,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
       %{test_file: test_file}
     end
 
-    test "strips / from filename", %{test_file: file} do
-      file = %Plug.Upload{file | filename: "../../../../../nested/bad.jpg"}
-      {:ok, %Object{} = object} = ActivityPub.upload(file)
-      [%{"href" => href}] = object.data["url"]
-      assert Regex.match?(~r"/bad.jpg$", href)
-      refute Regex.match?(~r"/nested/", href)
-    end
-
     test "sets a description if given", %{test_file: file} do
       {:ok, %Object{} = object} = ActivityPub.upload(file, description: "a cool file")
       assert object.data["name"] == "a cool file"

test/pleroma/web/pleroma_api/controllers/mascot_controller_test.exs

@@ -64,6 +64,10 @@ defmodule Pleroma.Web.PleromaAPI.MascotControllerTest do
 
     assert json_response_and_validate_schema(ret_conn, 200)
 
+    %{"url" => uploaded_url} = Jason.decode!(ret_conn.resp_body)
+
+    assert uploaded_url != nil and is_binary(uploaded_url)
+
     user = User.get_cached_by_id(user.id)
 
     conn =
@@ -72,6 +76,6 @@ defmodule Pleroma.Web.PleromaAPI.MascotControllerTest do
       |> get("/api/v1/pleroma/mascot")
 
     assert %{"url" => url, "type" => "image"} = json_response_and_validate_schema(conn, 200)
-    assert url =~ "an_image"
+    assert url == uploaded_url
   end
 end

test/test_helper.exs

@@ -9,6 +9,10 @@ Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, :manual)
 
 {:ok, _} = Application.ensure_all_started(:ex_machina)
 
+# Prepare and later automatically cleanup upload dir
+uploads_dir = Pleroma.Config.get([Pleroma.Uploaders.Local, :uploads], "test/uploads")
+File.mkdir_p!(uploads_dir)
+
 ExUnit.after_suite(fn _results ->
   uploads = Pleroma.Config.get([Pleroma.Uploaders.Local, :uploads], "test/uploads")
   File.rm_rf!(uploads)