Merge branch 'develop' of https://git.pleroma.social/pleroma/pleroma into develop

.gitattributes

@@ -1,2 +1,10 @@
 *.ex diff=elixir
 *.exs diff=elixir
+
+priv/static/instance/static.css diff=css
+
+# Most of js/css files included in the repo are minified bundles,
+#   and we don't want to search/diff those as text files.
+*.js binary
+*.js.map binary
+*.css binary

.gitignore

@@ -4,6 +4,7 @@ secret
 /db
 /deps
 /*.ez
+/test/instance
 /test/uploads
 /.elixir_ls
 /test/fixtures/DSCN0010_tmp.jpg

.gitlab-ci.yml

@@ -57,7 +57,7 @@ unit-testing:
     policy: pull
 
   services:
-  - name: postgres:9.6
+  - name: postgres:13
     alias: postgres
     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
   script:
@@ -228,8 +228,8 @@ arm:
   artifacts: *release-artifacts
   only: *release-only
   tags:
-    - arm32
+    - arm32-specified
-  image: elixir:1.10.3
+  image: arm32v7/elixir:1.10.3
   cache: *release-cache
   variables: *release-variables
   before_script: *before-release
@@ -240,8 +240,8 @@ arm-musl:
   artifacts: *release-artifacts
   only: *release-only
   tags:
-    - arm32
+    - arm32-specified
-  image: elixir:1.10.3-alpine
+  image: arm32v7/elixir:1.10.3-alpine
   cache: *release-cache
   variables: *release-variables
   before_script: *before-release-musl
@@ -253,7 +253,7 @@ arm64:
   only: *release-only
   tags:
     - arm
-  image: elixir:1.10.3
+  image: arm64v8/elixir:1.10.3
   cache: *release-cache
   variables: *release-variables
   before_script: *before-release
@@ -265,8 +265,7 @@ arm64-musl:
   only: *release-only
   tags:
     - arm
-  # TODO: Replace with upstream image when 1.9.0 comes out
+  image: arm64v8/elixir:1.10.3-alpine
-  image: elixir:1.10.3-alpine
   cache: *release-cache
   variables: *release-variables
   before_script: *before-release-musl

CHANGELOG.md

@@ -1,53 +1,110 @@
 # Changelog
+
 All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
+### Changed
+
+- Polls now always return a `voters_count`, even if they are single-choice.
+- Admin Emails: The ap id is used as the user link in emails now.
+- Improved registration workflow for email confirmation and account approval modes.
+- **Breaking:** Changed `mix pleroma.user toggle_confirmed` to `mix pleroma.user confirm`
+- Search: When using Postgres 11+, Pleroma will use the `websearch_to_tsvector` function to parse search queries.
+- Emoji: Support the full Unicode 13.1 set of Emoji for reactions, plus regional indicators.
+
 ### Added
-- Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`)
+
-- Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`)
-- Mix task option for force-unfollowing relays
-- Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details).
 - Reports now generate notifications for admins and mods.
-- Pleroma API: Importing the mutes users from CSV files.
 - Experimental websocket-based federation between Pleroma instances.
-- Support pagination of blocks and mutes
+- Support for local-only statuses.
-- App metrics: ability to restrict access to specified IP whitelist.
+- Support pagination of blocks and mutes.
-- Account backup
+- Account backup.
 - Configuration: Add `:instance, autofollowing_nicknames` setting to provide a way to make accounts automatically follow new users that register on the local Pleroma instance.
-- Ability to view remote timelines, with ex. `/api/v1/timelines/public?instance=lain.com` and streams `public:remote` and `public:remote:media`
+- Ability to view remote timelines, with ex. `/api/v1/timelines/public?instance=lain.com` and streams `public:remote` and `public:remote:media`.
 - The site title is now injected as a `title` tag like preloads or metadata.
+- Password reset tokens now are not accepted after a certain age.
+- Mix tasks to help with displaying and removing ConfigDB entries. See `mix pleroma.config`.
+- OAuth form improvements: users are remembered by their cookie, the CSS is overridable by the admin, and the style has been improved.
+- OAuth improvements and fixes: more secure session-based authentication (by token that could be revoked anytime), ability to revoke belonging OAuth token from any client etc.
+
+<details>
+  <summary>API Changes</summary>
+- Admin API: (`GET /api/pleroma/admin/users`) filter users by `unconfirmed` status and `actor_type`.
+- Pleroma API: Add `idempotency_key` to the chat message entity that can be used for optimistic message sending.
+- Pleroma API: (`GET /api/v1/pleroma/federation_status`) Add a way to get a list of unreachable instances.
+- Mastodon API: User and conversation mutes can now auto-expire if `expires_in` parameter was given while adding the mute.
+- Admin API: An endpoint to manage frontends.
+- Streaming API: Add follow relationships updates.
+</details>
+
+### Fixed
+
+- Users with `is_discoverable` field set to false (default value) will appear in in-service search results but be hidden from external services (search bots etc.).
+- Streaming API: Posts and notifications are not dropped, when CLI task is executing.
+- Creating incorrect IPv4 address-style HTTP links when encountering certain numbers.
+
+<details>
+  <summary>API Changes</summary>
+  - Mastodon API: Current user is now included in conversation if it's the only participant.
+  - Mastodon API: Fixed last_status.account being not filled with account data.
+</details>
+
+## Unreleased (Patch)
+
+### Fixed
+
+- Fix ability to update Pleroma Chat push notifications with PUT /api/v1/push/subscription and alert type pleroma:chat_mention
+- Emoji Reaction activity filtering from blocked and muted accounts.
+
+## [2.2.1] - 2020-12-22
+
+### Changed
+- Updated Pleroma FE
+
+### Fixed
+
+- Config generation: rename `Pleroma.Upload.Filter.ExifTool` to `Pleroma.Upload.Filter.Exiftool`.
+- S3 Uploads with Elixir 1.11.
+- Mix task pleroma.user delete_activities for source installations.
+- Search: RUM index search speed has been fixed.
+- Rich Media Previews sometimes showed the wrong preview due to a bug following redirects.
+- Fixes for the autolinker.
+- Forwarded reports duplication from Pleroma instances.
+
+- <details>
+    <summary>API</summary>
+  - Statuses were not displayed for Mastodon forwarded reports.
+  </details>
+
+### Upgrade notes
+
+1. Restart Pleroma
+
+## [2.2.0] - 2020-11-12
+
+### Security
+
+- Fixed the possibility of using file uploads to spoof posts.
 
 ### Changed
 
 - **Breaking** Requires `libmagic` (or `file`) to guess file types.
+- **Breaking:** App metrics endpoint (`/api/pleroma/app_metrics`) is disabled by default, check `docs/API/prometheus.md` on enabling and configuring.
 - **Breaking:** Pleroma Admin API: emoji packs and files routes changed.
 - **Breaking:** Sensitive/NSFW statuses no longer disable link previews.
-- **Breaking:** App metrics endpoint (`/api/pleroma/app_metrics`) is disabled by default, check `docs/API/prometheus.md` on enabling and configuring. 
 - Search: Users are now findable by their urls.
 - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated.
 - Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated.
 - The `discoverable` field in the `User` struct will now add a NOINDEX metatag to profile pages when false.
-- Users with the `discoverable` field set to false will not show up in searches.
+- Users with the `is_discoverable` field set to false will not show up in searches ([bug](https://git.pleroma.social/pleroma/pleroma/-/issues/2301)).
 - Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option).
 - Introduced optional dependencies on `ffmpeg`, `ImageMagick`, `exiftool` software packages. Please refer to `docs/installation/optional/media_graphics_packages.md`.
-- Polls now always return a `voters_count`, even if they are single-choice
+- <details>
-- Admin Emails: The ap id is used as the user link in emails now.
-
-<details>
   <summary>API Changes</summary>
-
+- API: Empty parameter values for integer parameters are now ignored in non-strict validaton mode.
-- Pleroma API: Importing the mutes users from CSV files.
-- Admin API: Importing emoji from a zip file
-- Pleroma API: Pagination for remote/local packs and emoji.
-- Admin API: (`GET /api/pleroma/admin/users`) added filters user by `unconfirmed` status
-- Admin API: (`GET /api/pleroma/admin/users`) added filters user by `actor_type`
-- Pleroma API: Add `idempotency_key` to the chat message entity that can be used for optimistic message sending.
-- Pleroma API: (`GET /api/v1/pleroma/federation_status`) Add a way to get a list of unreachable instances.
-- Mastodon API: User and conversation mutes can now auto-expire if `expires_in` parameter was given while adding the mute.
-
 </details>
 
 ### Removed
@@ -58,23 +115,38 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Removed `:managed_config` option. In practice, it was accidentally removed with 2.0.0 release when frontends were
 switched to a new configuration mechanism, however it was not officially removed until now.
 
+### Added
+
+- Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details).
+- Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`)
+- Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email resend_confirmation_emails`)
+- Mix task option for force-unfollowing relays
+- App metrics: ability to restrict access to specified IP whitelist.
+
+<details>
+  <summary>API Changes</summary>
+
+- Admin API: Importing emoji from a zip file
+- Pleroma API: Importing the mutes users from CSV files.
+- Pleroma API: Pagination for remote/local packs and emoji.
+
+</details>
+
 ### Fixed
 
 - Add documented-but-missing chat pagination.
 - Allow sending out emails again.
-- Allow sending chat messages to yourself.
+- Allow sending chat messages to yourself
-- Fix remote users with a whitespace name.
 - OStatus / static FE endpoints: fixed inaccessibility for anonymous users on non-federating instances, switched to handling per `:restrict_unauthenticated` setting.
-- Mastodon API: Current user is now included in conversation if it's the only participant
+- Fix remote users with a whitespace name.
-- Mastodon API: Fixed last_status.account being not filled with account data
 
-## Unreleased (Patch)
+### Upgrade notes
 
-### Changed
+1. Install libmagic and development headers (`libmagic-dev` on Ubuntu/Debian, `file-dev` on Alpine Linux)
-- API: Empty parameter values for integer parameters are now ignored in non-strict validaton mode.
+2. Run database migrations (inside Pleroma directory):
-
+  - OTP: `./bin/pleroma_ctl migrate`
-### Fixes
+  - From Source: `mix ecto.migrate`
-- Config generation: rename `Pleroma.Upload.Filter.ExifTool` to `Pleroma.Upload.Filter.Exiftool`
+3. Restart Pleroma
 
 ## [2.1.2] - 2020-09-17
 

Dockerfile

@@ -31,9 +31,9 @@ LABEL maintainer="ops@pleroma.social" \
 ARG HOME=/opt/pleroma
 ARG DATA=/var/lib/pleroma
 
-RUN echo "https://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories &&\
+RUN echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories &&\
 	apk update &&\
-	apk add exiftool imagemagick ncurses postgresql-client &&\
+	apk add exiftool imagemagick libmagic ncurses postgresql-client &&\
 	adduser --system --shell /bin/false --home ${HOME} pleroma &&\
 	mkdir -p ${DATA}/uploads &&\
 	mkdir -p ${DATA}/static &&\

SECURITY.md

@@ -6,7 +6,7 @@ Currently, Pleroma offers bugfixes and security patches only for the latest mino
 
 | Version | Support 
 |---------| --------
-| 2.1     | Bugfixes and security patches
+| 2.2     | Bugfixes and security patches
 
 ## Reporting a vulnerability
 

benchmarks/load_testing/users.ex

@@ -109,8 +109,8 @@ defmodule Pleroma.LoadTesting.Users do
   end
 
   def make_friends(%User{} = main_user, %User{} = user) do
-    {:ok, _} = User.follow(main_user, user)
+    {:ok, _, _} = User.follow(main_user, user)
-    {:ok, _} = User.follow(user, main_user)
+    {:ok, _, _} = User.follow(user, main_user)
   end
 
   @spec get_users(User.t(), keyword()) :: [User.t()]

benchmarks/mix/tasks/pleroma/benchmarks/timelines.ex

@@ -50,7 +50,7 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Timelines do
     )
 
     users
-    |> Enum.each(fn {:ok, follower} -> Pleroma.User.follow(follower, user) end)
+    |> Enum.each(fn {:ok, follower, user} -> Pleroma.User.follow(follower, user) end)
 
     Benchee.run(
       %{

config/config.exs

@@ -47,7 +47,6 @@ use Mix.Config
 config :pleroma, ecto_repos: [Pleroma.Repo]
 
 config :pleroma, Pleroma.Repo,
-  types: Pleroma.PostgresTypes,
   telemetry_event: [Pleroma.Repo.Instrumenter],
   migration_lock: nil
 
@@ -129,7 +128,6 @@ config :pleroma, Pleroma.Web.Endpoint,
     dispatch: [
       {:_,
        [
-         {"/api/fedsocket/v1", Pleroma.Web.FedSockets.IncomingHandler, []},
          {"/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, []},
          {"/websocket", Phoenix.Endpoint.CowboyWebSocket,
           {Phoenix.Transports.WebSocket,
@@ -148,16 +146,6 @@ config :pleroma, Pleroma.Web.Endpoint,
     "SameSite=Lax"
   ]
 
-config :pleroma, :fed_sockets,
-  enabled: false,
-  connection_duration: :timer.hours(8),
-  rejection_duration: :timer.minutes(15),
-  fed_socket_fetches: [
-    default: 12_000,
-    interval: 3_000,
-    lazy: false
-  ]
-
 # Configures Elixir's Logger
 config :logger, :console,
   level: :debug,
@@ -267,7 +255,8 @@ config :pleroma, :instance,
       length: 16
     ]
   ],
-  show_reactions: true
+  show_reactions: true,
+  password_reset_token_validity: 60 * 60 * 24
 
 config :pleroma, :welcome,
   direct_message: [
@@ -319,7 +308,7 @@ config :pleroma, :frontend_configurations,
     hideSitename: false,
     hideUserStats: false,
     loginMethod: "password",
-    logo: "/static/logo.png",
+    logo: "/static/logo.svg",
     logoMargin: ".1em",
     logoMask: true,
     minimalScopesMode: false,
@@ -356,8 +345,8 @@ config :pleroma, :assets,
 config :pleroma, :manifest,
   icons: [
     %{
-      src: "/static/logo.png",
+      src: "/static/logo.svg",
-      type: "image/png"
+      type: "image/svg+xml"
     }
   ],
   theme_color: "#282c37",
@@ -661,7 +650,7 @@ config :pleroma, :email_notifications,
   }
 
 config :pleroma, :oauth2,
-  token_expires_in: 600,
+  token_expires_in: 3600 * 24 * 365 * 100,
   issue_new_refresh_token: true,
   clean_expired_tokens: false
 

config/description.exs

@@ -272,19 +272,6 @@ config :pleroma, :config_description, [
       }
     ]
   },
-  %{
-    group: :pleroma,
-    key: :fed_sockets,
-    type: :group,
-    description: "Websocket based federation",
-    children: [
-      %{
-        key: :enabled,
-        type: :boolean,
-        description: "Enable FedSockets"
-      }
-    ]
-  },
   %{
     group: :pleroma,
     key: Pleroma.Emails.Mailer,
@@ -1267,7 +1254,7 @@ config :pleroma, :config_description, [
             hideSitename: false,
             hideUserStats: false,
             loginMethod: "password",
-            logo: "/static/logo.png",
+            logo: "/static/logo.svg",
             logoMargin: ".1em",
             logoMask: true,
             minimalScopesMode: false,
@@ -1353,7 +1340,7 @@ config :pleroma, :config_description, [
             key: :logo,
             type: {:string, :image},
             description: "URL of the logo, defaults to Pleroma's logo",
-            suggestions: ["/static/logo.png"]
+            suggestions: ["/static/logo.svg"]
           },
           %{
             key: :logoMargin,
@@ -1966,14 +1953,8 @@ config :pleroma, :config_description, [
     group: :pleroma,
     key: Oban,
     type: :group,
-    description: """
+    description:
-    [Oban](https://github.com/sorentwo/oban) asynchronous job processor configuration.
+      "[Oban](https://github.com/sorentwo/oban) asynchronous job processor configuration.",
-
-    Note: if you are running PostgreSQL in [`silent_mode`](https://postgresqlco.nf/en/doc/param/silent_mode?version=9.1),
-      it's advised to set [`log_destination`](https://postgresqlco.nf/en/doc/param/log_destination?version=9.1) to `syslog`,
-      otherwise `postmaster.log` file may grow because of "you don't own a lock of type ShareLock" warnings
-      (see https://github.com/sorentwo/oban/issues/52).
-    """,
     children: [
       %{
         key: :log,
@@ -2553,7 +2534,7 @@ config :pleroma, :config_description, [
         key: :token_expires_in,
         type: :integer,
         description: "The lifetime in seconds of the access token",
-        suggestions: [600]
+        suggestions: [2_592_000]
       },
       %{
         key: :issue_new_refresh_token,

config/releases.exs

@@ -1,31 +0,0 @@
-import Config
-
-config :pleroma, :instance, static_dir: "/var/lib/pleroma/static"
-config :pleroma, Pleroma.Uploaders.Local, uploads: "/var/lib/pleroma/uploads"
-config :pleroma, :modules, runtime_dir: "/var/lib/pleroma/modules"
-
-config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs"
-
-config :pleroma, release: true, config_path: config_path
-
-if File.exists?(config_path) do
-  import_config config_path
-else
-  warning = [
-    IO.ANSI.red(),
-    IO.ANSI.bright(),
-    "!!! #{config_path} not found! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file",
-    IO.ANSI.reset()
-  ]
-
-  IO.puts(warning)
-end
-
-exported_config =
-  config_path
-  |> Path.dirname()
-  |> Path.join("prod.exported_from_db.secret.exs")
-
-if File.exists?(exported_config) do
-  import_config exported_config
-end

config/test.exs

@@ -19,11 +19,6 @@ config :logger, :console,
   level: :warn,
   format: "\n[$level] $message\n"
 
-config :pleroma, :fed_sockets,
-  enabled: false,
-  connection_duration: 5,
-  rejection_duration: 5
-
 config :pleroma, :auth, oauth_consumer_strategies: []
 
 config :pleroma, Pleroma.Upload,
@@ -52,7 +47,10 @@ config :pleroma, Pleroma.Repo,
   password: "postgres",
   database: "pleroma_test",
   hostname: System.get_env("DB_HOST") || "localhost",
-  pool: Ecto.Adapters.SQL.Sandbox
+  pool: Ecto.Adapters.SQL.Sandbox,
+  pool_size: 50
+
+config :pleroma, :dangerzone, override_repo_pool_size: true
 
 # Reduce hash rounds for testing
 config :pbkdf2_elixir, rounds: 1
@@ -126,6 +124,16 @@ config :tzdata, :autoupdate, :disabled
 
 config :pleroma, :mrf, policies: []
 
+config :pleroma, :pipeline,
+  object_validator: Pleroma.Web.ActivityPub.ObjectValidatorMock,
+  mrf: Pleroma.Web.ActivityPub.MRFMock,
+  activity_pub: Pleroma.Web.ActivityPub.ActivityPubMock,
+  side_effects: Pleroma.Web.ActivityPub.SideEffectsMock,
+  federator: Pleroma.Web.FederatorMock,
+  config: Pleroma.ConfigMock
+
+config :pleroma, :cachex, provider: Pleroma.CachexMock
+
 if File.exists?("./config/test.secret.exs") do
   import_config "test.secret.exs"
 else

docs/API/admin_api.md

@@ -554,7 +554,7 @@ Response:
   * `show_role`
   * `skip_thread_containment`
   * `fields`
-  * `discoverable`
+  * `is_discoverable`
   * `actor_type`
 
 * Responses:
@@ -1499,3 +1499,66 @@ Returns the content of the document
   "url": "https://example.com/instance/panel.html"
 }
 ```
+
+## `GET /api/pleroma/admin/frontends
+
+### List available frontends
+
+- Response:
+
+```json
+[
+   {
+    "build_url": "https://git.pleroma.social/pleroma/fedi-fe/-/jobs/artifacts/${ref}/download?job=build",
+    "git": "https://git.pleroma.social/pleroma/fedi-fe",
+    "installed": true,
+    "name": "fedi-fe",
+    "ref": "master"
+  },
+  {
+    "build_url": "https://git.pleroma.social/lambadalambda/kenoma/-/jobs/artifacts/${ref}/download?job=build",
+    "git": "https://git.pleroma.social/lambadalambda/kenoma",
+    "installed": false,
+    "name": "kenoma",
+    "ref": "master"
+  }
+]
+```
+
+## `POST /api/pleroma/admin/frontends/install`
+
+### Install a frontend
+
+- Params:
+  - `name`: frontend name, required
+  - `ref`: frontend ref
+  - `file`: path to a frontend zip file
+  - `build_url`: build URL
+  - `build_dir`: build directory
+
+- Response:
+
+```json
+[
+   {
+    "build_url": "https://git.pleroma.social/pleroma/fedi-fe/-/jobs/artifacts/${ref}/download?job=build",
+    "git": "https://git.pleroma.social/pleroma/fedi-fe",
+    "installed": true,
+    "name": "fedi-fe",
+    "ref": "master"
+  },
+  {
+    "build_url": "https://git.pleroma.social/lambadalambda/kenoma/-/jobs/artifacts/${ref}/download?job=build",
+    "git": "https://git.pleroma.social/lambadalambda/kenoma",
+    "installed": false,
+    "name": "kenoma",
+    "ref": "master"
+  }
+]
+```
+
+```json
+{
+  "error": "Could not install frontend"
+}
+```

docs/API/differences_in_mastoapi_responses.md

@@ -4,7 +4,7 @@ A Pleroma instance can be identified by "<Mastodon version> (compatible; Pleroma
 
 ## Flake IDs
 
-Pleroma uses 128-bit ids as opposed to Mastodon's 64 bits. However just like Mastodon's ids they are lexically sortable strings
+Pleroma uses 128-bit ids as opposed to Mastodon's 64 bits. However, just like Mastodon's ids, they are lexically sortable strings
 
 ## Timelines
 
@@ -18,7 +18,7 @@ Adding the parameter `instance=lain.com` to the public timeline will show only s
 
 ## Statuses
 
-- `visibility`: has an additional possible value `list`
+- `visibility`: has additional possible values `list` and `local` (for local-only statuses)
 
 Has these additional fields under the `pleroma` object:
 
@@ -26,8 +26,8 @@ Has these additional fields under the `pleroma` object:
 - `conversation_id`: the ID of the AP context the status is associated with (if any)
 - `direct_conversation_id`: the ID of the Mastodon direct message conversation the status is associated with (if any)
 - `in_reply_to_account_acct`: the `acct` property of User entity for replied user (if any)
-- `content`: a map consisting of alternate representations of the `content` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain`
+- `content`: a map consisting of alternate representations of the `content` property with the key being its mimetype. Currently, the only alternate representation supported is `text/plain`
-- `spoiler_text`: a map consisting of alternate representations of the `spoiler_text` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain`
+- `spoiler_text`: a map consisting of alternate representations of the `spoiler_text` property with the key being its mimetype. Currently, the only alternate representation supported is `text/plain`
 - `expires_at`: a datetime (iso8601) that states when the post will expire (be deleted automatically), or empty if the post won't expire
 - `thread_muted`: true if the thread the post belongs to is muted
 - `emoji_reactions`: A list with emoji / reaction maps. The format is `{name: "☕", count: 1, me: true}`. Contains no information about the reacting users, for that use the `/statuses/:id/reactions` endpoint.
@@ -84,7 +84,7 @@ Has these additional fields under the `pleroma` object:
 
 - `show_role`: boolean, nullable, true when the user wants his role (e.g admin, moderator) to be shown
 - `no_rich_text` - boolean, nullable, true when html tags are stripped from all statuses requested from the API
-- `discoverable`: boolean, true when the user allows discovery of the account in search results and other services.
+- `discoverable`: boolean, true when the user allows external services (search bots) etc. to index / list the account (regardless of this setting, user will still appear in regular search results)
 - `actor_type`: string, the type of this account.
 
 ## Conversations
@@ -170,10 +170,10 @@ Returns on success: 200 OK `{}`
 
 Additional parameters can be added to the JSON body/Form data:
 
-- `preview`: boolean, if set to `true` the post won't be actually posted, but the status entitiy would still be rendered back. This could be useful for previewing rich text/custom emoji, for example.
+- `preview`: boolean, if set to `true` the post won't be actually posted, but the status entity would still be rendered back. This could be useful for previewing rich text/custom emoji, for example.
 - `content_type`: string, contain the MIME type of the status, it is transformed into HTML by the backend. You can get the list of the supported MIME types with the nodeinfo endpoint.
-- `to`: A list of nicknames (like `lain@soykaf.club` or `lain` on the local server) that will be used to determine who is going to be addressed by this post. Using this will disable the implicit addressing by mentioned names in the `status` body, only the people in the `to` list will be addressed. The normal rules for for post visibility are not affected by this and will still apply.
+- `to`: A list of nicknames (like `lain@soykaf.club` or `lain` on the local server) that will be used to determine who is going to be addressed by this post. Using this will disable the implicit addressing by mentioned names in the `status` body, only the people in the `to` list will be addressed. The normal rules for post visibility are not affected by this and will still apply.
-- `visibility`: string, besides standard MastoAPI values (`direct`, `private`, `unlisted` or `public`) it can be used to address a List by setting it to `list:LIST_ID`.
+- `visibility`: string, besides standard MastoAPI values (`direct`, `private`, `unlisted`, `local` or `public`) it can be used to address a List by setting it to `list:LIST_ID`.
 - `expires_in`: The number of seconds the posted activity should expire in. When a posted activity expires it will be deleted from the server, and a delete request for it will be federated. This needs to be longer than an hour.
 - `in_reply_to_conversation_id`: Will reply to a given conversation, addressing only the people who are part of the recipient set of that conversation. Sets the visibility to `direct`.
 
@@ -207,7 +207,7 @@ Additional parameters can be added to the JSON body/Form data:
 - `skip_thread_containment` - if true, skip filtering out broken threads
 - `allow_following_move` - if true, allows automatically follow moved following accounts
 - `pleroma_background_image` - sets the background image of the user. Can be set to "" (an empty string) to reset.
-- `discoverable` - if true, discovery of this account in search results and other services is allowed.
+- `discoverable` - if true, external services (search bots) etc. are allowed to index / list the account (regardless of this setting, user will still appear in regular search results).
 - `actor_type` - the type of this account.
 - `accepts_chat_messages` - if false, this account will reject all chat messages.
 
@@ -233,7 +233,7 @@ Post here request with `grant_type=refresh_token` to obtain new access token. Re
 
 `POST /api/v1/accounts`
 
-Has theses additional parameters (which are the same as in Pleroma-API):
+Has these additional parameters (which are the same as in Pleroma-API):
 
 - `fullname`: optional
 - `bio`: optional
@@ -261,6 +261,16 @@ Has theses additional parameters (which are the same as in Pleroma-API):
 - `pleroma.metadata.post_formats`: A list of the allowed post format types
 - `vapid_public_key`: The public key needed for push messages
 
+## Push Subscription
+
+`POST /api/v1/push/subscription`
+`PUT /api/v1/push/subscription`
+
+Permits these additional alert types:
+
+- pleroma:chat_mention
+- pleroma:emoji_reaction
+
 ## Markers
 
 Has these additional fields under the `pleroma` object:
@@ -269,10 +279,27 @@ Has these additional fields under the `pleroma` object:
 
 ## Streaming
 
+### Chats
+
 There is an additional `user:pleroma_chat` stream. Incoming chat messages will make the current chat be sent to this `user` stream. The `event` of an incoming chat message is `pleroma:chat_update`. The payload is the updated chat with the incoming chat message in the `last_message` field.
 
+### Remote timelines
+
 For viewing remote server timelines, there are `public:remote` and `public:remote:media` streams. Each of these accept a parameter like `?instance=lain.com`.
 
+### Follow relationships updates
+
+Pleroma streams follow relationships updates as `pleroma:follow_relationships_update` events to the `user` stream.
+
+The message payload consist of:
+
+- `state`: a relationship state, one of `follow_pending`, `follow_accept` or `follow_reject`.
+
+- `follower` and `following` maps with following fields:
+  - `id`: user ID
+  - `follower_count`: follower count
+  - `following_count`: following count
+
 ## User muting and thread muting
 
 Both user muting and thread muting can be done for only a certain time by adding an `expires_in` parameter to the API calls and giving the expiration time in seconds.

docs/API/pleroma_api.md

@@ -579,14 +579,14 @@ Emoji reactions work a lot like favourites do. They make it possible to react to
 ### React to a post with a unicode emoji
 * Method: `PUT`
 * Authentication: required
-* Params: `emoji`: A single character unicode emoji
+* Params: `emoji`: A unicode RGI emoji or a regional indicator
 * Response: JSON, the status.
 
 ## `DELETE /api/v1/pleroma/statuses/:id/reactions/:emoji`
 ### Remove a reaction to a post with a unicode emoji
 * Method: `DELETE`
 * Authentication: required
-* Params: `emoji`: A single character unicode emoji
+* Params: `emoji`: A unicode RGI emoji or a regional indicator
 * Response: JSON, the status.
 
 ## `GET /api/v1/pleroma/statuses/:id/reactions`

docs/administration/CLI_tasks/config.md

@@ -32,7 +32,7 @@
     config :pleroma, configurable_from_database: false
     ```
 
-To delete transfered settings from database optional flag `-d` can be used. `<env>` is `prod` by default.
+To delete transferred settings from database optional flag `-d` can be used. `<env>` is `prod` by default.
 
 === "OTP"
     ```sh
@@ -43,3 +43,111 @@ To delete transfered settings from database optional flag `-d` can be used. `<en
     ```sh
     mix pleroma.config migrate_from_db [--env=<env>] [-d]
     ```
+
+## Dump all of the config settings defined in the database
+
+=== "OTP"
+
+    ```sh
+     ./bin/pleroma_ctl config dump
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.config dump
+    ```
+
+## List individual configuration groups in the database
+
+=== "OTP"
+
+    ```sh
+     ./bin/pleroma_ctl config groups
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.config groups
+    ```
+
+## Dump the saved configuration values for a specific group or key
+
+e.g., this shows all the settings under `config :pleroma`
+
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl config dump pleroma
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.config dump pleroma
+    ```
+
+To get values under a specific key:
+
+e.g., this shows all the settings under `config :pleroma, :instance`
+
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl config dump pleroma instance
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.config dump pleroma instance
+    ```
+
+## Delete the saved configuration values for a specific group or key
+
+e.g., this deletes all the settings under `config :tesla`
+
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl config delete [--force] tesla
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.config delete [--force] tesla
+    ```
+
+To delete values under a specific key:
+
+e.g., this deletes all the settings under `config :phoenix, :stacktrace_depth`
+
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl config delete [--force] phoenix stacktrace_depth
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.config delete [--force] phoenix stacktrace_depth
+    ```
+
+## Remove all settings from the database
+
+This forcibly removes all saved values in the database.
+
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl config [--force] reset
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.config [--force] reset
+    ```

docs/administration/CLI_tasks/email.md

@@ -16,7 +16,6 @@
     mix pleroma.email test [--to <destination email address>]
     ```
 
-
 Example:
 
 === "OTP"
@@ -36,11 +35,11 @@ Example:
 === "OTP"
 
     ```sh
-     ./bin/pleroma_ctl email send_confirmation_mails
+     ./bin/pleroma_ctl email resend_confirmation_emails
     ```
 
 === "From Source"
 
     ```sh
-    mix pleroma.email send_confirmation_mails
+    mix pleroma.email resend_confirmation_emails
     ```

docs/administration/CLI_tasks/user.md

@@ -264,13 +264,13 @@
 === "OTP"
 
     ```sh
-     ./bin/pleroma_ctl user toggle_confirmed <nickname>
+     ./bin/pleroma_ctl user confirm <nickname>
     ```
 
 === "From Source"
 
     ```sh
-    mix pleroma.user toggle_confirmed <nickname>
+    mix pleroma.user confirm <nickname>
     ```
 
 ## Set confirmation status for all regular active users

docs/configuration/cheatsheet.md

@@ -63,6 +63,7 @@ To add configuration to your config file, you can copy it from the base config.
 * `external_user_synchronization`: Enabling following/followers counters synchronization for external users.
 * `cleanup_attachments`: Remove attachments along with statuses. Does not affect duplicate files and attachments without status. Enabling this will increase load to database when deleting statuses on larger instances.
 * `show_reactions`: Let favourites and emoji reactions be viewed through the API (default: `true`).
+* `password_reset_token_validity`: The time after which reset tokens aren't accepted anymore, in seconds (default: one day).
 
 ## Welcome
 * `direct_message`: - welcome message sent as a direct message.
@@ -220,18 +221,6 @@ config :pleroma, :mrf_user_allowlist, %{
 * `total_user_limit`: the number of scheduled activities a user is allowed to create in total (Default: `300`)
 * `enabled`: whether scheduled activities are sent to the job queue to be executed
 
-## FedSockets
-FedSockets is an experimental feature allowing for Pleroma backends to federate using a persistant websocket connection as opposed to making each federation a seperate http connection. This feature is currently off by default. It is configurable throught he following options.
-
-### :fedsockets
-* `enabled`: Enables FedSockets for this instance. `false` by default.
-* `connection_duration`: Time an idle websocket is kept open.
-* `rejection_duration`: Failures to connect via FedSockets will not be retried for this period of time.
-* `fed_socket_fetches` and `fed_socket_rejections`: Settings passed to `cachex` for the fetch registry, and rejection stacks. See `Pleroma.Web.FedSockets` for more details.
-
-
-## Frontends
-
 ### :frontend_configurations
 
 This can be used to configure a keyword list that keeps the configuration data for any kind of frontend. By default, settings for `pleroma_fe` and `masto_fe` are configured. You can find the documentation for `pleroma_fe` configuration into [Pleroma-FE configuration and customization for instance administrators](/frontend/CONFIGURATION/#options).

docs/configuration/howto_database_config.md

@@ -5,7 +5,7 @@ The configuration of Pleroma has traditionally been managed with a config file,
 
 ## Migration to database config
 
-1. Run the mix task to migrate to the database. You'll receive some debugging output and a few messages informing you of what happened.
+1. Run the mix task to migrate to the database.
 
   **Source:**
 
@@ -24,21 +24,8 @@ The configuration of Pleroma has traditionally been managed with a config file,
   ```
 
   ```
-   10:04:34.155 [debug] QUERY OK source="config" db=1.6ms decode=2.0ms queue=33.5ms idle=0.0ms
-    SELECT c0."id", c0."key", c0."group", c0."value", c0."inserted_at", c0."updated_at" FROM "config" AS c0 []
     Migrating settings from file: /home/pleroma/config/dev.secret.exs
 
-   10:04:34.240 [debug] QUERY OK db=4.5ms queue=0.3ms idle=92.2ms
-    TRUNCATE config; []
-  
-   10:04:34.244 [debug] QUERY OK db=2.8ms queue=0.3ms idle=97.2ms
-    ALTER SEQUENCE config_id_seq RESTART; []
-    
-   10:04:34.256 [debug] QUERY OK source="config" db=0.8ms queue=1.4ms idle=109.8ms
-   SELECT c0."id", c0."key", c0."group", c0."value", c0."inserted_at", c0."updated_at" FROM "config" AS c0 WHERE ((c0."group" = $1) AND (c0."key" = $2)) [":pleroma", ":instance"]
-    
-   10:04:34.292 [debug] QUERY OK db=2.6ms queue=1.7ms idle=137.7ms
-   INSERT INTO "config" ("group","key","value","inserted_at","updated_at") VALUES ($1,$2,$3,$4,$5) RETURNING "id" [":pleroma", ":instance", <<131, 108, 0, 0, 0, 1, 104, 2, 100, 0, 4, 110, 97, 109, 101, 109, 0, 0, 0, 7, 66, 108, 101, 114, 111, 109, 97, 106>>, ~N[2020-07-12 15:04:34], ~N[2020-07-12 15:04:34]]
    Settings for key instance migrated.
    Settings for group :pleroma migrated.
   ```
@@ -124,12 +111,20 @@ The configuration of Pleroma has traditionally been managed with a config file,
 ## Debugging
 
 ### Clearing database config
-You can clear the database config by truncating the `config` table in the database. e.g.,
+You can clear the database config with the following command:
+
+  **Source:**
 
   ```
-psql -d pleroma_dev
+  $ mix pleroma.config reset
-pleroma_dev=# TRUNCATE config;
+  ```
-TRUNCATE TABLE
+
+  or
+
+  **OTP:**
+
+  ```
+  $ ./bin/pleroma_ctl config reset
   ```
 
 Additionally, every time you migrate the configuration to the database the config table is automatically truncated to ensure a clean migration.
@@ -137,17 +132,24 @@ Additionally, every time you migrate the configuration to the database the confi
 ### Manually removing a setting
 If you encounter a situation where the server cannot run properly because of an invalid setting in the database and this is preventing you from accessing AdminFE, you can manually remove the offending setting if you know which one it is.
 
-e.g., here is an example showing a minimal configuration in the database. Only the `config :pleroma, :instance` settings are in the table:
+e.g., here is an example showing a the removal of the `config :pleroma, :instance` settings:
+
+  **Source:**
 
   ```
-psql -d pleroma_dev
+  $ mix pleroma.config delete pleroma instance
-pleroma_dev=# select * from config;
+  Are you sure you want to continue? [n]  y
- id |    key    |                           value                            |     inserted_at     |     updated_at      |  group
+  config :pleroma, :instance deleted from the ConfigDB.
-----+-----------+------------------------------------------------------------+---------------------+---------------------+----------
+  ```
-  1 | :instance | \x836c0000000168026400046e616d656d00000007426c65726f6d616a | 2020-07-12 15:33:29 | 2020-07-12 15:33:29 | :pleroma
+
-(1 row)
+  or
-pleroma_dev=# delete from config where key = ':instance' and group = ':pleroma';
+
-DELETE 1
+  **OTP:**
+
+  ```
+  $ ./bin/pleroma_ctl config delete pleroma instance
+  Are you sure you want to continue? [n]  y
+  config :pleroma, :instance deleted from the ConfigDB.
   ```
 
 Now the `config :pleroma, :instance` settings have been removed from the database.

docs/configuration/static_dir.md

@@ -88,3 +88,8 @@ config :pleroma, :frontend_configurations,
     Note the extra `static` folder for the terms-of-service.html
 
 Terms of Service will be shown to all users on the registration page. It's the best place where to write down the rules for your instance. You can modify the rules by adding and changing `$static_dir/static/terms-of-service.html`.
+
+ 	
+## Styling rendered pages
+
+To overwrite the CSS stylesheet of the OAuth form and other static pages, you can upload your own CSS file to `instance/static/static.css`. This will completely replace the CSS used by those pages, so it might be a good idea to copy the one from `priv/static/instance/static.css` and make your changes.

docs/dev.md

@@ -14,9 +14,9 @@ This document contains notes and guidelines for Pleroma developers.
 
   For `:api` pipeline routes, it'll be verified whether `OAuthScopesPlug` was called or explicitly skipped, and if it was not then auth information will be dropped for request. Then `EnsurePublicOrAuthenticatedPlug` will be called to ensure that either the instance is not private or user is authenticated (unless explicitly skipped). Such automated checks help to prevent human errors and result in higher security / privacy for users.
 
-## [HTTP Basic Authentication](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Authorization)
+## Non-OAuth authentication
 
-* With HTTP Basic Auth, OAuth scopes check is _not_ performed for any action (since password is provided during the auth, requester is able to obtain a token with full permissions anyways). `Pleroma.Web.Plugs.AuthenticationPlug` and `Pleroma.Web.Plugs.LegacyAuthenticationPlug` both call `Pleroma.Web.Plugs.OAuthScopesPlug.skip_plug(conn)` when password is provided.
+* With non-OAuth authentication ([HTTP Basic Authentication](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Authorization) or HTTP header- or params-provided auth), OAuth scopes check is _not_ performed for any action (since password is provided during the auth, requester is able to obtain a token with full permissions anyways); auth plugs invoke `Pleroma.Helpers.AuthHelper.skip_oauth(conn)` in this case.
 
 ## Auth-related configuration, OAuth consumer mode etc.
 

docs/installation/debian_based_en.md

@@ -35,7 +35,7 @@ sudo apt full-upgrade
 * Install some of the above mentioned programs:
 
 ```shell
-sudo apt install git build-essential postgresql postgresql-contrib cmake libmagic-devel
+sudo apt install git build-essential postgresql postgresql-contrib cmake libmagic-dev
 ```
 
 ### Install Elixir and Erlang

installation/download-mastofe-build.sh

@@ -9,29 +9,32 @@ static_dir="instance/static"
 # project_branch="pleroma"
 # static_dir="priv/static"
 
-if [[ ! -d "${static_dir}" ]]
+if [ ! -d "${static_dir}" ]
 then
 	echo "Error: ${static_dir} directory is missing, are you sure you are running this script at the root of pleroma’s repository?"
 	exit 1
 fi
 
-last_modified="$(curl -s -I 'https://git.pleroma.social/api/v4/projects/'${project_id}'/jobs/artifacts/'${project_branch}'/download?job=build' | grep '^Last-Modified:' | cut -d: -f2-)"
+last_modified="$(curl --fail -s -I 'https://git.pleroma.social/api/v4/projects/'${project_id}'/jobs/artifacts/'${project_branch}'/download?job=build' | grep '^Last-Modified:' | cut -d: -f2-)"
 
 echo "branch:${project_branch}"
 echo "Last-Modified:${last_modified}"
 
 artifact="mastofe.zip"
 
-if [[ -e mastofe.timestamp ]] && [[ "${last_modified}" != "" ]]
+if [ "${last_modified}x" = "x" ]
 then
-	if [[ "$(cat mastofe.timestamp)" == "${last_modified}" ]]
+	echo "ERROR: Couldn't get the modification date of the latest build archive, maybe it expired, exiting..."
-	then
+	exit 1
-		echo "MastoFE is up-to-date, exiting…"
-		exit 0
-	fi
 fi
 
-curl -c - "https://git.pleroma.social/api/v4/projects/${project_id}/jobs/artifacts/${project_branch}/download?job=build" -o "${artifact}" || exit
+if [ -e mastofe.timestamp ] && [ "$(cat mastofe.timestamp)" = "${last_modified}" ]
+then
+	echo "MastoFE is up-to-date, exiting..."
+	exit 0
+fi
+
+curl --fail -c - "https://git.pleroma.social/api/v4/projects/${project_id}/jobs/artifacts/${project_branch}/download?job=build" -o "${artifact}" || exit
 
 # TODO: Update the emoji as well
 rm -fr "${static_dir}/sw.js" "${static_dir}/packs" || exit

installation/pleroma.nginx

@@ -93,9 +93,4 @@ server {
         chunked_transfer_encoding on;
         proxy_pass         http://phoenix;
     }
-
-    location /api/fedsocket/v1 {
-        proxy_request_buffering off;
-        proxy_pass http://phoenix/api/fedsocket/v1;
-    }
 }

lib/mix/pleroma.ex

@@ -12,17 +12,19 @@ defmodule Mix.Pleroma do
     :cachex,
     :flake_id,
     :swoosh,
-    :timex
+    :timex,
+    :fast_html
   ]
-  @cachex_children ["object", "user", "scrubber"]
+  @cachex_children ["object", "user", "scrubber", "web_resp"]
   @doc "Common functions to be reused in mix tasks"
   def start_pleroma do
     Pleroma.Config.Holder.save_default()
     Pleroma.Config.Oban.warn()
+    Pleroma.Application.limiters_setup()
     Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
 
-    if Pleroma.Config.get(:env) != :test do
+    unless System.get_env("DEBUG") do
-      Application.put_env(:logger, :console, level: :debug)
+      Logger.remove_backend(:console)
     end
 
     adapter = Application.get_env(:tesla, :adapter)
@@ -36,12 +38,23 @@ defmodule Mix.Pleroma do
 
     Enum.each(apps, &Application.ensure_all_started/1)
 
+    oban_config = [
+      crontab: [],
+      repo: Pleroma.Repo,
+      log: false,
+      queues: [],
+      plugins: []
+    ]
+
     children =
       [
         Pleroma.Repo,
+        Pleroma.Emoji,
         {Pleroma.Config.TransferTask, false},
         Pleroma.Web.Endpoint,
-        {Oban, Pleroma.Config.get(Oban)}
+        {Oban, oban_config},
+        {Majic.Pool,
+         [name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]}
       ] ++
         http_children(adapter)
 
@@ -97,12 +110,6 @@ defmodule Mix.Pleroma do
     end
   end
 
-  def shell_yes?(message) do
-    if mix_shell?(),
-      do: Mix.shell().yes?("Continue?"),
-      else: shell_prompt(message, "Continue?") in ~w(Yn Y y)
-  end
-
   def shell_info(message) do
     if mix_shell?(),
       do: Mix.shell().info(message),

lib/mix/tasks/pleroma/config.ex

@@ -5,6 +5,7 @@
 defmodule Mix.Tasks.Pleroma.Config do
   use Mix.Task
 
+  import Ecto.Query
   import Mix.Pleroma
 
   alias Pleroma.ConfigDB
@@ -14,11 +15,14 @@ defmodule Mix.Tasks.Pleroma.Config do
   @moduledoc File.read!("docs/administration/CLI_tasks/config.md")
 
   def run(["migrate_to_db"]) do
+    check_configdb(fn ->
       start_pleroma()
       migrate_to_db()
+    end)
   end
 
   def run(["migrate_from_db" | options]) do
+    check_configdb(fn ->
       start_pleroma()
 
       {opts, _} =
@@ -28,12 +32,182 @@ defmodule Mix.Tasks.Pleroma.Config do
         )
 
       migrate_from_db(opts)
+    end)
+  end
+
+  def run(["dump"]) do
+    check_configdb(fn ->
+      start_pleroma()
+
+      header = config_header()
+
+      settings =
+        ConfigDB
+        |> Repo.all()
+        |> Enum.sort()
+
+      unless settings == [] do
+        shell_info("#{header}")
+
+        Enum.each(settings, &dump(&1))
+      else
+        shell_error("No settings in ConfigDB.")
+      end
+    end)
+  end
+
+  def run(["dump", group, key]) do
+    check_configdb(fn ->
+      start_pleroma()
+
+      group = maybe_atomize(group)
+      key = maybe_atomize(key)
+
+      group
+      |> ConfigDB.get_by_group_and_key(key)
+      |> dump()
+    end)
+  end
+
+  def run(["dump", group]) do
+    check_configdb(fn ->
+      start_pleroma()
+
+      group = maybe_atomize(group)
+
+      dump_group(group)
+    end)
+  end
+
+  def run(["groups"]) do
+    check_configdb(fn ->
+      start_pleroma()
+
+      groups =
+        ConfigDB
+        |> distinct([c], true)
+        |> select([c], c.group)
+        |> Repo.all()
+
+      if length(groups) > 0 do
+        shell_info("The following configuration groups are set in ConfigDB:\r\n")
+        groups |> Enum.each(fn x -> shell_info("-  #{x}") end)
+        shell_info("\r\n")
+      end
+    end)
+  end
+
+  def run(["reset", "--force"]) do
+    check_configdb(fn ->
+      start_pleroma()
+      truncatedb()
+      shell_info("The ConfigDB settings have been removed from the database.")
+    end)
+  end
+
+  def run(["reset"]) do
+    check_configdb(fn ->
+      start_pleroma()
+
+      shell_info("The following settings will be permanently removed:")
+
+      ConfigDB
+      |> Repo.all()
+      |> Enum.sort()
+      |> Enum.each(&dump(&1))
+
+      shell_error("\nTHIS CANNOT BE UNDONE!")
+
+      if shell_prompt("Are you sure you want to continue?", "n") in ~w(Yn Y y) do
+        truncatedb()
+
+        shell_info("The ConfigDB settings have been removed from the database.")
+      else
+        shell_error("No changes made.")
+      end
+    end)
+  end
+
+  def run(["delete", "--force", group, key]) do
+    start_pleroma()
+
+    group = maybe_atomize(group)
+    key = maybe_atomize(key)
+
+    with true <- key_exists?(group, key) do
+      shell_info("The following settings will be removed from ConfigDB:\n")
+
+      group
+      |> ConfigDB.get_by_group_and_key(key)
+      |> dump()
+
+      delete_key(group, key)
+    else
+      _ ->
+        shell_error("No settings in ConfigDB for #{inspect(group)}, #{inspect(key)}. Aborting.")
+    end
+  end
+
+  def run(["delete", "--force", group]) do
+    start_pleroma()
+
+    group = maybe_atomize(group)
+
+    with true <- group_exists?(group) do
+      shell_info("The following settings will be removed from ConfigDB:\n")
+      dump_group(group)
+      delete_group(group)
+    else
+      _ -> shell_error("No settings in ConfigDB for #{inspect(group)}. Aborting.")
+    end
+  end
+
+  def run(["delete", group, key]) do
+    start_pleroma()
+
+    group = maybe_atomize(group)
+    key = maybe_atomize(key)
+
+    with true <- key_exists?(group, key) do
+      shell_info("The following settings will be removed from ConfigDB:\n")
+
+      group
+      |> ConfigDB.get_by_group_and_key(key)
+      |> dump()
+
+      if shell_prompt("Are you sure you want to continue?", "n") in ~w(Yn Y y) do
+        delete_key(group, key)
+      else
+        shell_error("No changes made.")
+      end
+    else
+      _ ->
+        shell_error("No settings in ConfigDB for #{inspect(group)}, #{inspect(key)}. Aborting.")
+    end
+  end
+
+  def run(["delete", group]) do
+    start_pleroma()
+
+    group = maybe_atomize(group)
+
+    with true <- group_exists?(group) do
+      shell_info("The following settings will be removed from ConfigDB:\n")
+      dump_group(group)
+
+      if shell_prompt("Are you sure you want to continue?", "n") in ~w(Yn Y y) do
+        delete_group(group)
+      else
+        shell_error("No changes made.")
+      end
+    else
+      _ -> shell_error("No settings in ConfigDB for #{inspect(group)}. Aborting.")
+    end
   end
 
   @spec migrate_to_db(Path.t() | nil) :: any()
   def migrate_to_db(file_path \\ nil) do
-    with true <- Pleroma.Config.get([:configurable_from_database]),
+    with :ok <- Pleroma.Config.DeprecationWarnings.warn() do
-         :ok <- Pleroma.Config.DeprecationWarnings.warn() do
       config_file =
         if file_path do
           file_path
@@ -47,16 +221,15 @@ defmodule Mix.Tasks.Pleroma.Config do
 
       do_migrate_to_db(config_file)
     else
-      :error -> deprecation_error()
+      _ ->
-      _ -> migration_error()
+        shell_error("Migration is not allowed until all deprecation warnings have been resolved.")
     end
   end
 
   defp do_migrate_to_db(config_file) do
     if File.exists?(config_file) do
       shell_info("Migrating settings from file: #{Path.expand(config_file)}")
-      Ecto.Adapters.SQL.query!(Repo, "TRUNCATE config;")
+      truncatedb()
-      Ecto.Adapters.SQL.query!(Repo, "ALTER SEQUENCE config_id_seq RESTART;")
 
       custom_config =
         config_file
@@ -80,11 +253,10 @@ defmodule Mix.Tasks.Pleroma.Config do
       shell_info("Settings for key #{key} migrated.")
     end)
 
-    shell_info("Settings for group :#{group} migrated.")
+    shell_info("Settings for group #{inspect(group)} migrated.")
   end
 
   defp migrate_from_db(opts) do
-    if Pleroma.Config.get([:configurable_from_database]) do
     env = opts[:env] || Pleroma.Config.get(:env)
 
     config_path =
@@ -111,19 +283,6 @@ defmodule Mix.Tasks.Pleroma.Config do
     shell_info(
       "Database configuration settings have been exported to config/#{env}.exported_from_db.secret.exs"
     )
-    else
-      migration_error()
-    end
-  end
-
-  defp migration_error do
-    shell_error(
-      "Migration is not allowed in config. You can change this behavior by setting `config :pleroma, configurable_from_database: true`"
-    )
-  end
-
-  defp deprecation_error do
-    shell_error("Migration is not allowed until all deprecation warnings have been resolved.")
   end
 
   if Code.ensure_loaded?(Config.Reader) do
@@ -150,8 +309,80 @@ defmodule Mix.Tasks.Pleroma.Config do
 
   defp delete(config, true) do
     {:ok, _} = Repo.delete(config)
-    shell_info("#{config.key} deleted from DB.")
+
+    shell_info(
+      "config #{inspect(config.group)}, #{inspect(config.key)} was deleted from the ConfigDB."
+    )
   end
 
   defp delete(_config, _), do: :ok
+
+  defp dump(%ConfigDB{} = config) do
+    value = inspect(config.value, limit: :infinity)
+
+    shell_info("config #{inspect(config.group)}, #{inspect(config.key)}, #{value}\r\n\r\n")
+  end
+
+  defp dump(_), do: :noop
+
+  defp dump_group(group) when is_atom(group) do
+    group
+    |> ConfigDB.get_all_by_group()
+    |> Enum.each(&dump/1)
+  end
+
+  defp group_exists?(group) do
+    group
+    |> ConfigDB.get_all_by_group()
+    |> Enum.any?()
+  end
+
+  defp key_exists?(group, key) do
+    group
+    |> ConfigDB.get_by_group_and_key(key)
+    |> is_nil
+    |> Kernel.!()
+  end
+
+  defp maybe_atomize(arg) when is_atom(arg), do: arg
+
+  defp maybe_atomize(":" <> arg), do: maybe_atomize(arg)
+
+  defp maybe_atomize(arg) when is_binary(arg) do
+    if ConfigDB.module_name?(arg) do
+      String.to_existing_atom("Elixir." <> arg)
+    else
+      String.to_atom(arg)
+    end
+  end
+
+  defp check_configdb(callback) do
+    with true <- Pleroma.Config.get([:configurable_from_database]) do
+      callback.()
+    else
+      _ ->
+        shell_error(
+          "ConfigDB not enabled. Please check the value of :configurable_from_database in your configuration."
+        )
+    end
+  end
+
+  defp delete_key(group, key) do
+    check_configdb(fn ->
+      ConfigDB.delete(%{group: group, key: key})
+    end)
+  end
+
+  defp delete_group(group) do
+    check_configdb(fn ->
+      group
+      |> ConfigDB.get_all_by_group()
+      |> Enum.each(&ConfigDB.delete/1)
+    end)
+  end
+
+  defp truncatedb do
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE config;")
+    Ecto.Adapters.SQL.query!(Repo, "ALTER SEQUENCE config_id_seq RESTART;")
+  end
 end

lib/mix/tasks/pleroma/database.ex

@@ -48,9 +48,15 @@ defmodule Mix.Tasks.Pleroma.Database do
   def run(["update_users_following_followers_counts"]) do
     start_pleroma()
 
-    User
+    Repo.transaction(
-    |> Repo.all()
+      fn ->
-    |> Enum.each(&User.update_follower_count/1)
+        from(u in User, select: u)
+        |> Repo.stream()
+        |> Stream.each(&User.update_follower_count/1)
+        |> Stream.run()
+      end,
+      timeout: :infinity
+    )
   end
 
   def run(["prune_objects" | args]) do

lib/mix/tasks/pleroma/frontend.ex

@@ -17,8 +17,6 @@ defmodule Mix.Tasks.Pleroma.Frontend do
   end
 
   def run(["install", frontend | args]) do
-    log_level = Logger.level()
-    Logger.configure(level: :warn)
     start_pleroma()
 
     {options, [], []} =
@@ -33,109 +31,6 @@ defmodule Mix.Tasks.Pleroma.Frontend do
         ]
       )
 
-    instance_static_dir =
+    Pleroma.Frontend.install(frontend, options)
-      with nil <- options[:static_dir] do
-        Pleroma.Config.get!([:instance, :static_dir])
-      end
-
-    cmd_frontend_info = %{
-      "name" => frontend,
-      "ref" => options[:ref],
-      "build_url" => options[:build_url],
-      "build_dir" => options[:build_dir]
-    }
-
-    config_frontend_info = Pleroma.Config.get([:frontends, :available, frontend], %{})
-
-    frontend_info =
-      Map.merge(config_frontend_info, cmd_frontend_info, fn _key, config, cmd ->
-        # This only overrides things that are actually set
-        cmd || config
-      end)
-
-    ref = frontend_info["ref"]
-
-    unless ref do
-      raise "No ref given or configured"
-    end
-
-    dest =
-      Path.join([
-        instance_static_dir,
-        "frontends",
-        frontend,
-        ref
-      ])
-
-    fe_label = "#{frontend} (#{ref})"
-
-    tmp_dir = Path.join([instance_static_dir, "frontends", "tmp"])
-
-    with {_, :ok} <-
-           {:download_or_unzip, download_or_unzip(frontend_info, tmp_dir, options[:file])},
-         shell_info("Installing #{fe_label} to #{dest}"),
-         :ok <- install_frontend(frontend_info, tmp_dir, dest) do
-      File.rm_rf!(tmp_dir)
-      shell_info("Frontend #{fe_label} installed to #{dest}")
-
-      Logger.configure(level: log_level)
-    else
-      {:download_or_unzip, _} ->
-        shell_info("Could not download or unzip the frontend")
-
-      _e ->
-        shell_info("Could not install the frontend")
-    end
-  end
-
-  defp download_or_unzip(frontend_info, temp_dir, file) do
-    if file do
-      with {:ok, zip} <- File.read(Path.expand(file)) do
-        unzip(zip, temp_dir)
-      end
-    else
-      download_build(frontend_info, temp_dir)
-    end
-  end
-
-  def unzip(zip, dest) do
-    with {:ok, unzipped} <- :zip.unzip(zip, [:memory]) do
-      File.rm_rf!(dest)
-      File.mkdir_p!(dest)
-
-      Enum.each(unzipped, fn {filename, data} ->
-        path = filename
-
-        new_file_path = Path.join(dest, path)
-
-        new_file_path
-        |> Path.dirname()
-        |> File.mkdir_p!()
-
-        File.write!(new_file_path, data)
-      end)
-
-      :ok
-    end
-  end
-
-  defp download_build(frontend_info, dest) do
-    shell_info("Downloading pre-built bundle for #{frontend_info["name"]}")
-    url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
-
-    with {:ok, %{status: 200, body: zip_body}} <-
-           Pleroma.HTTP.get(url, [], pool: :media, recv_timeout: 120_000) do
-      unzip(zip_body, dest)
-    else
-      e -> {:error, e}
-    end
-  end
-
-  defp install_frontend(frontend_info, source, dest) do
-    from = frontend_info["build_dir"] || "dist"
-    File.rm_rf!(dest)
-    File.mkdir_p!(dest)
-    File.cp_r!(Path.join([source, from]), dest)
-    :ok
   end
 end

lib/mix/tasks/pleroma/instance.ex

@@ -161,12 +161,21 @@ defmodule Mix.Tasks.Pleroma.Instance do
         )
         |> Path.expand()
 
+      {strip_uploads_message, strip_uploads_default} =
+        if Pleroma.Utils.command_available?("exiftool") do
+          {"Do you want to strip location (GPS) data from uploaded images? This requires exiftool, it was detected as installed. (y/n)",
+           "y"}
+        else
+          {"Do you want to strip location (GPS) data from uploaded images? This requires exiftool, it was detected as not installed, please install it if you answer yes. (y/n)",
+           "n"}
+        end
+
       strip_uploads =
         get_option(
           options,
           :strip_uploads,
-          "Do you want to strip location (GPS) data from uploaded images? (y/n)",
+          strip_uploads_message,
-          "y"
+          strip_uploads_default
         ) === "y"
 
       anonymize_uploads =
@@ -253,7 +262,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
     else
       shell_error(
         "The task would have overwritten the following files:\n" <>
-          (Enum.map(paths, &"- #{&1}\n") |> Enum.join("")) <>
+          (Enum.map(will_overwrite, &"- #{&1}\n") |> Enum.join("")) <>
           "Rerun with `--force` to overwrite them."
       )
     end

lib/mix/tasks/pleroma/user.ex

@@ -60,7 +60,7 @@ defmodule Mix.Tasks.Pleroma.User do
       - admin: #{if(admin?, do: "true", else: "false")}
     """)
 
-    proceed? = assume_yes? or shell_yes?("Continue?")
+    proceed? = assume_yes? or shell_prompt("Continue?", "n") in ~w(Yn Y y)
 
     if proceed? do
       start_pleroma()
@@ -378,11 +378,11 @@ defmodule Mix.Tasks.Pleroma.User do
     end
   end
 
-  def run(["toggle_confirmed", nickname]) do
+  def run(["confirm", nickname]) do
     start_pleroma()
 
     with %User{} = user <- User.get_cached_by_nickname(nickname) do
-      {:ok, user} = User.toggle_confirmation(user)
+      {:ok, user} = User.confirm(user)
 
       message = if user.confirmation_pending, do: "needs", else: "doesn't need"
 

lib/pleroma/activity.ex

@@ -24,6 +24,8 @@ defmodule Pleroma.Activity do
 
   @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true}
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
   schema "activities" do
     field(:data, :map)
     field(:local, :boolean, default: true)
@@ -194,6 +196,19 @@ defmodule Pleroma.Activity do
     end
   end
 
+  def get_by_id_with_user_actor(id) do
+    case FlakeId.flake_id?(id) do
+      true ->
+        Activity
+        |> where([a], a.id == ^id)
+        |> with_preloaded_user_actor()
+        |> Repo.one()
+
+      _ ->
+        nil
+    end
+  end
+
   def get_by_id_with_object(id) do
     Activity
     |> where(id: ^id)
@@ -285,7 +300,7 @@ defmodule Pleroma.Activity do
 
   defp purge_web_resp_cache(%Activity{} = activity) do
     %{path: path} = URI.parse(activity.data["id"])
-    Cachex.del(:web_resp_cache, path)
+    @cachex.del(:web_resp_cache, path)
     activity
   end
 
@@ -356,4 +371,15 @@ defmodule Pleroma.Activity do
     actor = user_actor(activity)
     activity.id in actor.pinned_activities
   end
+
+  @spec get_by_object_ap_id_with_object(String.t()) :: t() | nil
+  def get_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do
+    ap_id
+    |> Queries.by_object_id()
+    |> with_preloaded_object()
+    |> first()
+    |> Repo.one()
+  end
+
+  def get_by_object_ap_id_with_object(_), do: nil
 end

lib/pleroma/activity/search.ex

@@ -19,15 +19,25 @@ defmodule Pleroma.Activity.Search do
     offset = Keyword.get(options, :offset, 0)
     author = Keyword.get(options, :author)
 
+    search_function =
+      if :persistent_term.get({Pleroma.Repo, :postgres_version}) >= 11 do
+        :websearch
+      else
+        :plain
+      end
+
     Activity
     |> Activity.with_preloaded_object()
     |> Activity.restrict_deactivated_users()
     |> restrict_public()
-    |> query_with(index_type, search_query)
+    |> query_with(index_type, search_query, search_function)
     |> maybe_restrict_local(user)
     |> maybe_restrict_author(author)
     |> maybe_restrict_blocked(user)
-    |> Pagination.fetch_paginated(%{"offset" => offset, "limit" => limit}, :offset)
+    |> Pagination.fetch_paginated(
+      %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum},
+      :offset
+    )
     |> maybe_fetch(user, search_query)
   end
 
@@ -50,7 +60,7 @@ defmodule Pleroma.Activity.Search do
     )
   end
 
-  defp query_with(q, :gin, search_query) do
+  defp query_with(q, :gin, search_query, :plain) do
     from([a, o] in q,
       where:
         fragment(
@@ -61,7 +71,18 @@ defmodule Pleroma.Activity.Search do
     )
   end
 
-  defp query_with(q, :rum, search_query) do
+  defp query_with(q, :gin, search_query, :websearch) do
+    from([a, o] in q,
+      where:
+        fragment(
+          "to_tsvector('english', ?->>'content') @@ websearch_to_tsquery('english', ?)",
+          o.data,
+          ^search_query
+        )
+    )
+  end
+
+  defp query_with(q, :rum, search_query, :plain) do
     from([a, o] in q,
       where:
         fragment(
@@ -73,6 +94,18 @@ defmodule Pleroma.Activity.Search do
     )
   end
 
+  defp query_with(q, :rum, search_query, :websearch) do
+    from([a, o] in q,
+      where:
+        fragment(
+          "? @@ websearch_to_tsquery('english', ?)",
+          o.fts_content,
+          ^search_query
+        ),
+      order_by: [fragment("? <=> now()::date", o.inserted_at)]
+    )
+  end
+
   defp maybe_restrict_local(q, user) do
     limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated)
 

lib/pleroma/application.ex

@@ -57,6 +57,7 @@ defmodule Pleroma.Application do
     setup_instrumenters()
     load_custom_modules()
     Pleroma.Docs.JSON.compile()
+    limiters_setup()
 
     adapter = Application.get_env(:tesla, :adapter)
 
@@ -109,7 +110,28 @@ defmodule Pleroma.Application do
     # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
     # for other strategies and supported options
     opts = [strategy: :one_for_one, name: Pleroma.Supervisor]
-    Supervisor.start_link(children, opts)
+    result = Supervisor.start_link(children, opts)
+
+    set_postgres_server_version()
+
+    result
+  end
+
+  defp set_postgres_server_version do
+    version =
+      with %{rows: [[version]]} <- Ecto.Adapters.SQL.query!(Pleroma.Repo, "show server_version"),
+           {num, _} <- Float.parse(version) do
+        num
+      else
+        e ->
+          Logger.warn(
+            "Could not get the postgres version: #{inspect(e)}.\nSetting the default value of 9.6"
+          )
+
+          9.6
+      end
+
+    :persistent_term.put({Pleroma.Repo, :postgres_version}, version)
   end
 
   def load_custom_modules do
@@ -207,8 +229,7 @@ defmodule Pleroma.Application do
          name: Pleroma.Web.Streamer.registry(),
          keys: :duplicate,
          partitions: System.schedulers_online()
-       ]},
+       ]}
-      Pleroma.Web.FedSockets.Supervisor
     ]
   end
 
@@ -273,4 +294,10 @@ defmodule Pleroma.Application do
   end
 
   defp http_children(_, _), do: []
+
+  @spec limiters_setup() :: :ok
+  def limiters_setup do
+    [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.MediaProxy]
+    |> Enum.each(&ConcurrentLimiter.new(&1, 1, 0))
+  end
 end

lib/pleroma/application_requirements.ex

@@ -24,6 +24,7 @@ defmodule Pleroma.ApplicationRequirements do
     |> check_migrations_applied!()
     |> check_welcome_message_config!()
     |> check_rum!()
+    |> check_repo_pool_size!()
     |> handle_result()
   end
 
@@ -188,6 +189,30 @@ defmodule Pleroma.ApplicationRequirements do
 
   defp check_system_commands!(result), do: result
 
+  defp check_repo_pool_size!(:ok) do
+    if Pleroma.Config.get([Pleroma.Repo, :pool_size], 10) != 10 and
+         not Pleroma.Config.get([:dangerzone, :override_repo_pool_size], false) do
+      Logger.error("""
+      !!!CONFIG WARNING!!!
+
+      The database pool size has been altered from the recommended value of 10.
+
+      Please revert or ensure your database is tuned appropriately and then set
+      `config :pleroma, :dangerzone, override_repo_pool_size: true`.
+
+      If you are experiencing database timeouts, please check the "Optimizing
+      your PostgreSQL performance" section in the documentation. If you still
+      encounter issues after that, please open an issue on the tracker.
+      """)
+
+      {:error, "Repo.pool_size different than recommended value."}
+    else
+      :ok
+    end
+  end
+
+  defp check_repo_pool_size!(result), do: result
+
   defp check_filter(filter, command_required) do
     filters = Config.get([Pleroma.Upload, :filters])
 

lib/pleroma/caching.ex

@@ -0,0 +1,19 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Caching do
+  @callback get!(Cachex.cache(), any()) :: any()
+  @callback get(Cachex.cache(), any()) :: {atom(), any()}
+  @callback put(Cachex.cache(), any(), any(), Keyword.t()) :: {Cachex.status(), boolean()}
+  @callback put(Cachex.cache(), any(), any()) :: {Cachex.status(), boolean()}
+  @callback fetch!(Cachex.cache(), any(), function() | nil) :: any()
+  # @callback del(Cachex.cache(), any(), Keyword.t()) :: {Cachex.status(), boolean()}
+  @callback del(Cachex.cache(), any()) :: {Cachex.status(), boolean()}
+  @callback stream!(Cachex.cache(), any()) :: Enumerable.t()
+  @callback expire_at(Cachex.cache(), binary(), number()) :: {Cachex.status(), boolean()}
+  @callback exists?(Cachex.cache(), any()) :: {Cachex.status(), boolean()}
+  @callback execute!(Cachex.cache(), function()) :: any()
+  @callback get_and_update(Cachex.cache(), any(), function()) ::
+              {:commit | :ignore, any()}
+end

lib/pleroma/captcha.ex

@@ -7,6 +7,8 @@ defmodule Pleroma.Captcha do
   alias Plug.Crypto.KeyGenerator
   alias Plug.Crypto.MessageEncryptor
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
   @doc """
   Ask the configured captcha service for a new captcha
   """
@@ -86,7 +88,7 @@ defmodule Pleroma.Captcha do
   end
 
   defp validate_usage(token) do
-    if is_nil(Cachex.get!(:used_captcha_cache, token)) do
+    if is_nil(@cachex.get!(:used_captcha_cache, token)) do
       :ok
     else
       {:error, :already_used}
@@ -95,7 +97,7 @@ defmodule Pleroma.Captcha do
 
   defp mark_captcha_as_used(token) do
     ttl = seconds_valid() |> :timer.seconds()
-    Cachex.put(:used_captcha_cache, token, true, ttl: ttl)
+    @cachex.put(:used_captcha_cache, token, true, ttl: ttl)
   end
 
   defp method, do: Pleroma.Config.get!([__MODULE__, :method])

lib/pleroma/config.ex

@@ -3,14 +3,18 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Config do
+  @behaviour Pleroma.Config.Getting
   defmodule Error do
     defexception [:message]
   end
 
+  @impl true
   def get(key), do: get(key, nil)
 
+  @impl true
   def get([key], default), do: get(key, default)
 
+  @impl true
   def get([_ | _] = path, default) do
     case fetch(path) do
       {:ok, value} -> value
@@ -18,6 +22,7 @@ defmodule Pleroma.Config do
     end
   end
 
+  @impl true
   def get(key, default) do
     Application.get_env(:pleroma, key, default)
   end

lib/pleroma/config/getting.ex

@@ -0,0 +1,8 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Config.Getting do
+  @callback get(any()) :: any()
+  @callback get(any(), any()) :: any()
+end

lib/pleroma/config/holder.ex

@@ -9,12 +9,7 @@ defmodule Pleroma.Config.Holder do
   def save_default do
     default_config =
       if System.get_env("RELEASE_NAME") do
-        release_config =
+        Pleroma.Config.Loader.merge(@config, release_defaults())
-          [:code.root_dir(), "releases", System.get_env("RELEASE_VSN"), "releases.exs"]
-          |> Path.join()
-          |> Pleroma.Config.Loader.read()
-
-        Pleroma.Config.Loader.merge(@config, release_config)
       else
         @config
       end
@@ -32,4 +27,16 @@ defmodule Pleroma.Config.Holder do
   def default_config(group, key), do: get_in(get_default(), [group, key])
 
   defp get_default, do: Pleroma.Config.get(:default_config)
+
+  @spec release_defaults() :: keyword()
+  def release_defaults do
+    [
+      pleroma: [
+        {:instance, [static_dir: "/var/lib/pleroma/static"]},
+        {Pleroma.Uploaders.Local, [uploads: "/var/lib/pleroma/uploads"]},
+        {:modules, [runtime_dir: "/var/lib/pleroma/modules"]},
+        {:release, true}
+      ]
+    ]
+  end
 end

lib/pleroma/config/release_runtime_provider.ex

@@ -0,0 +1,50 @@
+defmodule Pleroma.Config.ReleaseRuntimeProvider do
+  @moduledoc """
+  Imports `runtime.exs` and `{env}.exported_from_db.secret.exs` for elixir releases.
+  """
+  @behaviour Config.Provider
+
+  @impl true
+  def init(opts), do: opts
+
+  @impl true
+  def load(config, _opts) do
+    with_defaults = Config.Reader.merge(config, Pleroma.Config.Holder.release_defaults())
+
+    config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs"
+
+    with_runtime_config =
+      if File.exists?(config_path) do
+        runtime_config = Config.Reader.read!(config_path)
+
+        with_defaults
+        |> Config.Reader.merge(pleroma: [config_path: config_path])
+        |> Config.Reader.merge(runtime_config)
+      else
+        warning = [
+          IO.ANSI.red(),
+          IO.ANSI.bright(),
+          "!!! #{config_path} not found! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file",
+          IO.ANSI.reset()
+        ]
+
+        IO.puts(warning)
+        with_defaults
+      end
+
+    exported_config_path =
+      config_path
+      |> Path.dirname()
+      |> Path.join("prod.exported_from_db.secret.exs")
+
+    with_exported =
+      if File.exists?(exported_config_path) do
+        exported_config = Config.Reader.read!(with_runtime_config)
+        Config.Reader.merge(with_runtime_config, exported_config)
+      else
+        with_runtime_config
+      end
+
+    with_exported
+  end
+end

lib/pleroma/config_db.ex

@@ -6,7 +6,7 @@ defmodule Pleroma.ConfigDB do
   use Ecto.Schema
 
   import Ecto.Changeset
-  import Ecto.Query, only: [select: 3]
+  import Ecto.Query, only: [select: 3, from: 2]
   import Pleroma.Web.Gettext
 
   alias __MODULE__
@@ -41,8 +41,18 @@ defmodule Pleroma.ConfigDB do
     end)
   end
 
+  @spec get_all_by_group(atom() | String.t()) :: [t()]
+  def get_all_by_group(group) do
+    from(c in ConfigDB, where: c.group == ^group) |> Repo.all()
+  end
+
+  @spec get_by_group_and_key(atom() | String.t(), atom() | String.t()) :: t() | nil
+  def get_by_group_and_key(group, key) do
+    get_by_params(%{group: group, key: key})
+  end
+
   @spec get_by_params(map()) :: ConfigDB.t() | nil
-  def get_by_params(params), do: Repo.get_by(ConfigDB, params)
+  def get_by_params(%{group: _, key: _} = params), do: Repo.get_by(ConfigDB, params)
 
   @spec changeset(ConfigDB.t(), map()) :: Changeset.t()
   def changeset(config, params \\ %{}) do

lib/pleroma/constants.ex

@@ -26,4 +26,6 @@ defmodule Pleroma.Constants do
     do:
       ~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc embed.js embed.css)
   )
+
+  def as_local_public, do: Pleroma.Web.base_url() <> "/#Public"
 end

lib/pleroma/emails/admin_email.ex

@@ -48,6 +48,9 @@ defmodule Pleroma.Emails.AdminEmail do
               status_url = Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, id)
               "<li><a href=\"#{status_url}\">#{status_url}</li>"
 
+            %{"id" => id} when is_binary(id) ->
+              "<li><a href=\"#{id}\">#{id}</li>"
+
             id when is_binary(id) ->
               "<li><a href=\"#{id}\">#{id}</li>"
           end)

lib/pleroma/emails/user_email.ex

@@ -93,6 +93,19 @@ defmodule Pleroma.Emails.UserEmail do
     |> html_body(html_body)
   end
 
+  def approval_pending_email(user) do
+    html_body = """
+    <h3>Awaiting Approval</h3>
+    <p>Your account at #{instance_name()} is being reviewed by staff. You will receive another email once your account is approved.</p>
+    """
+
+    new()
+    |> to(recipient(user))
+    |> from(sender())
+    |> subject("Your account is awaiting approval")
+    |> html_body(html_body)
+  end
+
   @doc """
   Email used in digest email notifications
   Includes Mentions and New Followers data
@@ -151,7 +164,7 @@ defmodule Pleroma.Emails.UserEmail do
 
       logo_path =
         if is_nil(logo) do
-          Path.join(:code.priv_dir(:pleroma), "static/static/logo.png")
+          Path.join(:code.priv_dir(:pleroma), "static/static/logo.svg")
         else
           Path.join(Config.get([:instance, :static_dir]), logo)
         end
@@ -162,7 +175,7 @@ defmodule Pleroma.Emails.UserEmail do
       |> subject("Your digest from #{instance_name()}")
       |> put_layout(false)
       |> render_body("digest.html", html_data)
-      |> attachment(Swoosh.Attachment.new(logo_path, filename: "logo.png", type: :inline))
+      |> attachment(Swoosh.Attachment.new(logo_path, filename: "logo.svg", type: :inline))
     end
   end
 

lib/pleroma/emoji-data.txt

@@ -1,769 +0,0 @@
-# emoji-data.txt
-# Date: 2019-01-15, 12:10:05 GMT
-# © 2019 Unicode®, Inc.
-# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
-# For terms of use, see http://www.unicode.org/terms_of_use.html
-#
-# Emoji Data for UTS #51
-# Version: 12.0
-#
-# For documentation and usage, see http://www.unicode.org/reports/tr51
-#
-# Format: 
-# <codepoint(s)> ; <property> # <comments> 
-# Note: there is no guarantee as to the structure of whitespace or comments
-#
-# Characters and sequences are listed in code point order. Users should be shown a more natural order.
-# See the CLDR collation order for Emoji.
-
-
-# ================================================
-
-# All omitted code points have Emoji=No 
-# @missing: 0000..10FFFF  ; Emoji ; No
-
-0023          ; Emoji                #  1.1  [1] (#️)       number sign
-002A          ; Emoji                #  1.1  [1] (*️)       asterisk
-0030..0039    ; Emoji                #  1.1 [10] (0️..9️)    digit zero..digit nine
-00A9          ; Emoji                #  1.1  [1] (©️)       copyright
-00AE          ; Emoji                #  1.1  [1] (®️)       registered
-203C          ; Emoji                #  1.1  [1] (‼️)       double exclamation mark
-2049          ; Emoji                #  3.0  [1] (⁉️)       exclamation question mark
-2122          ; Emoji                #  1.1  [1] (™️)       trade mark
-2139          ; Emoji                #  3.0  [1] (ℹ️)       information
-2194..2199    ; Emoji                #  1.1  [6] (↔️..↙️)    left-right arrow..down-left arrow
-21A9..21AA    ; Emoji                #  1.1  [2] (↩️..↪️)    right arrow curving left..left arrow curving right
-231A..231B    ; Emoji                #  1.1  [2] (⌚..⌛)    watch..hourglass done
-2328          ; Emoji                #  1.1  [1] (⌨️)       keyboard
-23CF          ; Emoji                #  4.0  [1] (⏏️)       eject button
-23E9..23F3    ; Emoji                #  6.0 [11] (⏩..⏳)    fast-forward button..hourglass not done
-23F8..23FA    ; Emoji                #  7.0  [3] (⏸️..⏺️)    pause button..record button
-24C2          ; Emoji                #  1.1  [1] (Ⓜ️)       circled M
-25AA..25AB    ; Emoji                #  1.1  [2] (▪️..▫️)    black small square..white small square
-25B6          ; Emoji                #  1.1  [1] (▶️)       play button
-25C0          ; Emoji                #  1.1  [1] (◀️)       reverse button
-25FB..25FE    ; Emoji                #  3.2  [4] (◻️..◾)    white medium square..black medium-small square
-2600..2604    ; Emoji                #  1.1  [5] (☀️..☄️)    sun..comet
-260E          ; Emoji                #  1.1  [1] (☎️)       telephone
-2611          ; Emoji                #  1.1  [1] (☑️)       check box with check
-2614..2615    ; Emoji                #  4.0  [2] (☔..☕)    umbrella with rain drops..hot beverage
-2618          ; Emoji                #  4.1  [1] (☘️)       shamrock
-261D          ; Emoji                #  1.1  [1] (☝️)       index pointing up
-2620          ; Emoji                #  1.1  [1] (☠️)       skull and crossbones
-2622..2623    ; Emoji                #  1.1  [2] (☢️..☣️)    radioactive..biohazard
-2626          ; Emoji                #  1.1  [1] (☦️)       orthodox cross
-262A          ; Emoji                #  1.1  [1] (☪️)       star and crescent
-262E..262F    ; Emoji                #  1.1  [2] (☮️..☯️)    peace symbol..yin yang
-2638..263A    ; Emoji                #  1.1  [3] (☸️..☺️)    wheel of dharma..smiling face
-2640          ; Emoji                #  1.1  [1] (♀️)       female sign
-2642          ; Emoji                #  1.1  [1] (♂️)       male sign
-2648..2653    ; Emoji                #  1.1 [12] (♈..♓)    Aries..Pisces
-265F..2660    ; Emoji                #  1.1  [2] (♟️..♠️)    chess pawn..spade suit
-2663          ; Emoji                #  1.1  [1] (♣️)       club suit
-2665..2666    ; Emoji                #  1.1  [2] (♥️..♦️)    heart suit..diamond suit
-2668          ; Emoji                #  1.1  [1] (♨️)       hot springs
-267B          ; Emoji                #  3.2  [1] (♻️)       recycling symbol
-267E..267F    ; Emoji                #  4.1  [2] (♾️..♿)    infinity..wheelchair symbol
-2692..2697    ; Emoji                #  4.1  [6] (⚒️..⚗️)    hammer and pick..alembic
-2699          ; Emoji                #  4.1  [1] (⚙️)       gear
-269B..269C    ; Emoji                #  4.1  [2] (⚛️..⚜️)    atom symbol..fleur-de-lis
-26A0..26A1    ; Emoji                #  4.0  [2] (⚠️..⚡)    warning..high voltage
-26AA..26AB    ; Emoji                #  4.1  [2] (⚪..⚫)    white circle..black circle
-26B0..26B1    ; Emoji                #  4.1  [2] (⚰️..⚱️)    coffin..funeral urn
-26BD..26BE    ; Emoji                #  5.2  [2] (⚽..⚾)    soccer ball..baseball
-26C4..26C5    ; Emoji                #  5.2  [2] (⛄..⛅)    snowman without snow..sun behind cloud
-26C8          ; Emoji                #  5.2  [1] (⛈️)       cloud with lightning and rain
-26CE          ; Emoji                #  6.0  [1] (⛎)       Ophiuchus
-26CF          ; Emoji                #  5.2  [1] (⛏️)       pick
-26D1          ; Emoji                #  5.2  [1] (⛑️)       rescue worker’s helmet
-26D3..26D4    ; Emoji                #  5.2  [2] (⛓️..⛔)    chains..no entry
-26E9..26EA    ; Emoji                #  5.2  [2] (⛩️..⛪)    shinto shrine..church
-26F0..26F5    ; Emoji                #  5.2  [6] (⛰️..⛵)    mountain..sailboat
-26F7..26FA    ; Emoji                #  5.2  [4] (⛷️..⛺)    skier..tent
-26FD          ; Emoji                #  5.2  [1] (⛽)       fuel pump
-2702          ; Emoji                #  1.1  [1] (✂️)       scissors
-2705          ; Emoji                #  6.0  [1] (✅)       check mark button
-2708..2709    ; Emoji                #  1.1  [2] (✈️..✉️)    airplane..envelope
-270A..270B    ; Emoji                #  6.0  [2] (✊..✋)    raised fist..raised hand
-270C..270D    ; Emoji                #  1.1  [2] (✌️..✍️)    victory hand..writing hand
-270F          ; Emoji                #  1.1  [1] (✏️)       pencil
-2712          ; Emoji                #  1.1  [1] (✒️)       black nib
-2714          ; Emoji                #  1.1  [1] (✔️)       check mark
-2716          ; Emoji                #  1.1  [1] (✖️)       multiplication sign
-271D          ; Emoji                #  1.1  [1] (✝️)       latin cross
-2721          ; Emoji                #  1.1  [1] (✡️)       star of David
-2728          ; Emoji                #  6.0  [1] (✨)       sparkles
-2733..2734    ; Emoji                #  1.1  [2] (✳️..✴️)    eight-spoked asterisk..eight-pointed star
-2744          ; Emoji                #  1.1  [1] (❄️)       snowflake
-2747          ; Emoji                #  1.1  [1] (❇️)       sparkle
-274C          ; Emoji                #  6.0  [1] (❌)       cross mark
-274E          ; Emoji                #  6.0  [1] (❎)       cross mark button
-2753..2755    ; Emoji                #  6.0  [3] (❓..❕)    question mark..white exclamation mark
-2757          ; Emoji                #  5.2  [1] (❗)       exclamation mark
-2763..2764    ; Emoji                #  1.1  [2] (❣️..❤️)    heart exclamation..red heart
-2795..2797    ; Emoji                #  6.0  [3] (➕..➗)    plus sign..division sign
-27A1          ; Emoji                #  1.1  [1] (➡️)       right arrow
-27B0          ; Emoji                #  6.0  [1] (➰)       curly loop
-27BF          ; Emoji                #  6.0  [1] (➿)       double curly loop
-2934..2935    ; Emoji                #  3.2  [2] (⤴️..⤵️)    right arrow curving up..right arrow curving down
-2B05..2B07    ; Emoji                #  4.0  [3] (⬅️..⬇️)    left arrow..down arrow
-2B1B..2B1C    ; Emoji                #  5.1  [2] (⬛..⬜)    black large square..white large square
-2B50          ; Emoji                #  5.1  [1] (⭐)       star
-2B55          ; Emoji                #  5.2  [1] (⭕)       hollow red circle
-3030          ; Emoji                #  1.1  [1] (〰️)       wavy dash
-303D          ; Emoji                #  3.2  [1] (〽️)       part alternation mark
-3297          ; Emoji                #  1.1  [1] (㊗️)       Japanese “congratulations” button
-3299          ; Emoji                #  1.1  [1] (㊙️)       Japanese “secret” button
-1F004         ; Emoji                #  5.1  [1] (🀄)       mahjong red dragon
-1F0CF         ; Emoji                #  6.0  [1] (🃏)       joker
-1F170..1F171  ; Emoji                #  6.0  [2] (🅰️..🅱️)    A button (blood type)..B button (blood type)
-1F17E         ; Emoji                #  6.0  [1] (🅾️)       O button (blood type)
-1F17F         ; Emoji                #  5.2  [1] (🅿️)       P button
-1F18E         ; Emoji                #  6.0  [1] (🆎)       AB button (blood type)
-1F191..1F19A  ; Emoji                #  6.0 [10] (🆑..🆚)    CL button..VS button
-1F1E6..1F1FF  ; Emoji                #  6.0 [26] (🇦..🇿)    regional indicator symbol letter a..regional indicator symbol letter z
-1F201..1F202  ; Emoji                #  6.0  [2] (🈁..🈂️)    Japanese “here” button..Japanese “service charge” button
-1F21A         ; Emoji                #  5.2  [1] (🈚)       Japanese “free of charge” button
-1F22F         ; Emoji                #  5.2  [1] (🈯)       Japanese “reserved” button
-1F232..1F23A  ; Emoji                #  6.0  [9] (🈲..🈺)    Japanese “prohibited” button..Japanese “open for business” button
-1F250..1F251  ; Emoji                #  6.0  [2] (🉐..🉑)    Japanese “bargain” button..Japanese “acceptable” button
-1F300..1F320  ; Emoji                #  6.0 [33] (🌀..🌠)    cyclone..shooting star
-1F321         ; Emoji                #  7.0  [1] (🌡️)       thermometer
-1F324..1F32C  ; Emoji                #  7.0  [9] (🌤️..🌬️)    sun behind small cloud..wind face
-1F32D..1F32F  ; Emoji                #  8.0  [3] (🌭..🌯)    hot dog..burrito
-1F330..1F335  ; Emoji                #  6.0  [6] (🌰..🌵)    chestnut..cactus
-1F336         ; Emoji                #  7.0  [1] (🌶️)       hot pepper
-1F337..1F37C  ; Emoji                #  6.0 [70] (🌷..🍼)    tulip..baby bottle
-1F37D         ; Emoji                #  7.0  [1] (🍽️)       fork and knife with plate
-1F37E..1F37F  ; Emoji                #  8.0  [2] (🍾..🍿)    bottle with popping cork..popcorn
-1F380..1F393  ; Emoji                #  6.0 [20] (🎀..🎓)    ribbon..graduation cap
-1F396..1F397  ; Emoji                #  7.0  [2] (🎖️..🎗️)    military medal..reminder ribbon
-1F399..1F39B  ; Emoji                #  7.0  [3] (🎙️..🎛️)    studio microphone..control knobs
-1F39E..1F39F  ; Emoji                #  7.0  [2] (🎞️..🎟️)    film frames..admission tickets
-1F3A0..1F3C4  ; Emoji                #  6.0 [37] (🎠..🏄)    carousel horse..person surfing
-1F3C5         ; Emoji                #  7.0  [1] (🏅)       sports medal
-1F3C6..1F3CA  ; Emoji                #  6.0  [5] (🏆..🏊)    trophy..person swimming
-1F3CB..1F3CE  ; Emoji                #  7.0  [4] (🏋️..🏎️)    person lifting weights..racing car
-1F3CF..1F3D3  ; Emoji                #  8.0  [5] (🏏..🏓)    cricket game..ping pong
-1F3D4..1F3DF  ; Emoji                #  7.0 [12] (🏔️..🏟️)    snow-capped mountain..stadium
-1F3E0..1F3F0  ; Emoji                #  6.0 [17] (🏠..🏰)    house..castle
-1F3F3..1F3F5  ; Emoji                #  7.0  [3] (🏳️..🏵️)    white flag..rosette
-1F3F7         ; Emoji                #  7.0  [1] (🏷️)       label
-1F3F8..1F3FF  ; Emoji                #  8.0  [8] (🏸..🏿)    badminton..dark skin tone
-1F400..1F43E  ; Emoji                #  6.0 [63] (🐀..🐾)    rat..paw prints
-1F43F         ; Emoji                #  7.0  [1] (🐿️)       chipmunk
-1F440         ; Emoji                #  6.0  [1] (👀)       eyes
-1F441         ; Emoji                #  7.0  [1] (👁️)       eye
-1F442..1F4F7  ; Emoji                #  6.0[182] (👂..📷)    ear..camera
-1F4F8         ; Emoji                #  7.0  [1] (📸)       camera with flash
-1F4F9..1F4FC  ; Emoji                #  6.0  [4] (📹..📼)    video camera..videocassette
-1F4FD         ; Emoji                #  7.0  [1] (📽️)       film projector
-1F4FF         ; Emoji                #  8.0  [1] (📿)       prayer beads
-1F500..1F53D  ; Emoji                #  6.0 [62] (🔀..🔽)    shuffle tracks button..downwards button
-1F549..1F54A  ; Emoji                #  7.0  [2] (🕉️..🕊️)    om..dove
-1F54B..1F54E  ; Emoji                #  8.0  [4] (🕋..🕎)    kaaba..menorah
-1F550..1F567  ; Emoji                #  6.0 [24] (🕐..🕧)    one o’clock..twelve-thirty
-1F56F..1F570  ; Emoji                #  7.0  [2] (🕯️..🕰️)    candle..mantelpiece clock
-1F573..1F579  ; Emoji                #  7.0  [7] (🕳️..🕹️)    hole..joystick
-1F57A         ; Emoji                #  9.0  [1] (🕺)       man dancing
-1F587         ; Emoji                #  7.0  [1] (🖇️)       linked paperclips
-1F58A..1F58D  ; Emoji                #  7.0  [4] (🖊️..🖍️)    pen..crayon
-1F590         ; Emoji                #  7.0  [1] (🖐️)       hand with fingers splayed
-1F595..1F596  ; Emoji                #  7.0  [2] (🖕..🖖)    middle finger..vulcan salute
-1F5A4         ; Emoji                #  9.0  [1] (🖤)       black heart
-1F5A5         ; Emoji                #  7.0  [1] (🖥️)       desktop computer
-1F5A8         ; Emoji                #  7.0  [1] (🖨️)       printer
-1F5B1..1F5B2  ; Emoji                #  7.0  [2] (🖱️..🖲️)    computer mouse..trackball
-1F5BC         ; Emoji                #  7.0  [1] (🖼️)       framed picture
-1F5C2..1F5C4  ; Emoji                #  7.0  [3] (🗂️..🗄️)    card index dividers..file cabinet
-1F5D1..1F5D3  ; Emoji                #  7.0  [3] (🗑️..🗓️)    wastebasket..spiral calendar
-1F5DC..1F5DE  ; Emoji                #  7.0  [3] (🗜️..🗞️)    clamp..rolled-up newspaper
-1F5E1         ; Emoji                #  7.0  [1] (🗡️)       dagger
-1F5E3         ; Emoji                #  7.0  [1] (🗣️)       speaking head
-1F5E8         ; Emoji                #  7.0  [1] (🗨️)       left speech bubble
-1F5EF         ; Emoji                #  7.0  [1] (🗯️)       right anger bubble
-1F5F3         ; Emoji                #  7.0  [1] (🗳️)       ballot box with ballot
-1F5FA         ; Emoji                #  7.0  [1] (🗺️)       world map
-1F5FB..1F5FF  ; Emoji                #  6.0  [5] (🗻..🗿)    mount fuji..moai
-1F600         ; Emoji                #  6.1  [1] (😀)       grinning face
-1F601..1F610  ; Emoji                #  6.0 [16] (😁..😐)    beaming face with smiling eyes..neutral face
-1F611         ; Emoji                #  6.1  [1] (😑)       expressionless face
-1F612..1F614  ; Emoji                #  6.0  [3] (😒..😔)    unamused face..pensive face
-1F615         ; Emoji                #  6.1  [1] (😕)       confused face
-1F616         ; Emoji                #  6.0  [1] (😖)       confounded face
-1F617         ; Emoji                #  6.1  [1] (😗)       kissing face
-1F618         ; Emoji                #  6.0  [1] (😘)       face blowing a kiss
-1F619         ; Emoji                #  6.1  [1] (😙)       kissing face with smiling eyes
-1F61A         ; Emoji                #  6.0  [1] (😚)       kissing face with closed eyes
-1F61B         ; Emoji                #  6.1  [1] (😛)       face with tongue
-1F61C..1F61E  ; Emoji                #  6.0  [3] (😜..😞)    winking face with tongue..disappointed face
-1F61F         ; Emoji                #  6.1  [1] (😟)       worried face
-1F620..1F625  ; Emoji                #  6.0  [6] (😠..😥)    angry face..sad but relieved face
-1F626..1F627  ; Emoji                #  6.1  [2] (😦..😧)    frowning face with open mouth..anguished face
-1F628..1F62B  ; Emoji                #  6.0  [4] (😨..😫)    fearful face..tired face
-1F62C         ; Emoji                #  6.1  [1] (😬)       grimacing face
-1F62D         ; Emoji                #  6.0  [1] (😭)       loudly crying face
-1F62E..1F62F  ; Emoji                #  6.1  [2] (😮..😯)    face with open mouth..hushed face
-1F630..1F633  ; Emoji                #  6.0  [4] (😰..😳)    anxious face with sweat..flushed face
-1F634         ; Emoji                #  6.1  [1] (😴)       sleeping face
-1F635..1F640  ; Emoji                #  6.0 [12] (😵..🙀)    dizzy face..weary cat
-1F641..1F642  ; Emoji                #  7.0  [2] (🙁..🙂)    slightly frowning face..slightly smiling face
-1F643..1F644  ; Emoji                #  8.0  [2] (🙃..🙄)    upside-down face..face with rolling eyes
-1F645..1F64F  ; Emoji                #  6.0 [11] (🙅..🙏)    person gesturing NO..folded hands
-1F680..1F6C5  ; Emoji                #  6.0 [70] (🚀..🛅)    rocket..left luggage
-1F6CB..1F6CF  ; Emoji                #  7.0  [5] (🛋️..🛏️)    couch and lamp..bed
-1F6D0         ; Emoji                #  8.0  [1] (🛐)       place of worship
-1F6D1..1F6D2  ; Emoji                #  9.0  [2] (🛑..🛒)    stop sign..shopping cart
-1F6D5         ; Emoji                # 12.0  [1] (🛕)       hindu temple
-1F6E0..1F6E5  ; Emoji                #  7.0  [6] (🛠️..🛥️)    hammer and wrench..motor boat
-1F6E9         ; Emoji                #  7.0  [1] (🛩️)       small airplane
-1F6EB..1F6EC  ; Emoji                #  7.0  [2] (🛫..🛬)    airplane departure..airplane arrival
-1F6F0         ; Emoji                #  7.0  [1] (🛰️)       satellite
-1F6F3         ; Emoji                #  7.0  [1] (🛳️)       passenger ship
-1F6F4..1F6F6  ; Emoji                #  9.0  [3] (🛴..🛶)    kick scooter..canoe
-1F6F7..1F6F8  ; Emoji                # 10.0  [2] (🛷..🛸)    sled..flying saucer
-1F6F9         ; Emoji                # 11.0  [1] (🛹)       skateboard
-1F6FA         ; Emoji                # 12.0  [1] (🛺)       auto rickshaw
-1F7E0..1F7EB  ; Emoji                # 12.0 [12] (🟠..🟫)    orange circle..brown square
-1F90D..1F90F  ; Emoji                # 12.0  [3] (🤍..🤏)    white heart..pinching hand
-1F910..1F918  ; Emoji                #  8.0  [9] (🤐..🤘)    zipper-mouth face..sign of the horns
-1F919..1F91E  ; Emoji                #  9.0  [6] (🤙..🤞)    call me hand..crossed fingers
-1F91F         ; Emoji                # 10.0  [1] (🤟)       love-you gesture
-1F920..1F927  ; Emoji                #  9.0  [8] (🤠..🤧)    cowboy hat face..sneezing face
-1F928..1F92F  ; Emoji                # 10.0  [8] (🤨..🤯)    face with raised eyebrow..exploding head
-1F930         ; Emoji                #  9.0  [1] (🤰)       pregnant woman
-1F931..1F932  ; Emoji                # 10.0  [2] (🤱..🤲)    breast-feeding..palms up together
-1F933..1F93A  ; Emoji                #  9.0  [8] (🤳..🤺)    selfie..person fencing
-1F93C..1F93E  ; Emoji                #  9.0  [3] (🤼..🤾)    people wrestling..person playing handball
-1F93F         ; Emoji                # 12.0  [1] (🤿)       diving mask
-1F940..1F945  ; Emoji                #  9.0  [6] (🥀..🥅)    wilted flower..goal net
-1F947..1F94B  ; Emoji                #  9.0  [5] (🥇..🥋)    1st place medal..martial arts uniform
-1F94C         ; Emoji                # 10.0  [1] (🥌)       curling stone
-1F94D..1F94F  ; Emoji                # 11.0  [3] (🥍..🥏)    lacrosse..flying disc
-1F950..1F95E  ; Emoji                #  9.0 [15] (🥐..🥞)    croissant..pancakes
-1F95F..1F96B  ; Emoji                # 10.0 [13] (🥟..🥫)    dumpling..canned food
-1F96C..1F970  ; Emoji                # 11.0  [5] (🥬..🥰)    leafy green..smiling face with hearts
-1F971         ; Emoji                # 12.0  [1] (🥱)       yawning face
-1F973..1F976  ; Emoji                # 11.0  [4] (🥳..🥶)    partying face..cold face
-1F97A         ; Emoji                # 11.0  [1] (🥺)       pleading face
-1F97B         ; Emoji                # 12.0  [1] (🥻)       sari
-1F97C..1F97F  ; Emoji                # 11.0  [4] (🥼..🥿)    lab coat..flat shoe
-1F980..1F984  ; Emoji                #  8.0  [5] (🦀..🦄)    crab..unicorn
-1F985..1F991  ; Emoji                #  9.0 [13] (🦅..🦑)    eagle..squid
-1F992..1F997  ; Emoji                # 10.0  [6] (🦒..🦗)    giraffe..cricket
-1F998..1F9A2  ; Emoji                # 11.0 [11] (🦘..🦢)    kangaroo..swan
-1F9A5..1F9AA  ; Emoji                # 12.0  [6] (🦥..🦪)    sloth..oyster
-1F9AE..1F9AF  ; Emoji                # 12.0  [2] (🦮..🦯)    guide dog..probing cane
-1F9B0..1F9B9  ; Emoji                # 11.0 [10] (🦰..🦹)    red hair..supervillain
-1F9BA..1F9BF  ; Emoji                # 12.0  [6] (🦺..🦿)    safety vest..mechanical leg
-1F9C0         ; Emoji                #  8.0  [1] (🧀)       cheese wedge
-1F9C1..1F9C2  ; Emoji                # 11.0  [2] (🧁..🧂)    cupcake..salt
-1F9C3..1F9CA  ; Emoji                # 12.0  [8] (🧃..🧊)    beverage box..ice cube
-1F9CD..1F9CF  ; Emoji                # 12.0  [3] (🧍..🧏)    person standing..deaf person
-1F9D0..1F9E6  ; Emoji                # 10.0 [23] (🧐..🧦)    face with monocle..socks
-1F9E7..1F9FF  ; Emoji                # 11.0 [25] (🧧..🧿)    red envelope..nazar amulet
-1FA70..1FA73  ; Emoji                # 12.0  [4] (🩰..🩳)    ballet shoes..shorts
-1FA78..1FA7A  ; Emoji                # 12.0  [3] (🩸..🩺)    drop of blood..stethoscope
-1FA80..1FA82  ; Emoji                # 12.0  [3] (🪀..🪂)    yo-yo..parachute
-1FA90..1FA95  ; Emoji                # 12.0  [6] (🪐..🪕)    ringed planet..banjo
-
-# Total elements: 1311
-
-# ================================================
-
-# All omitted code points have Emoji_Presentation=No 
-# @missing: 0000..10FFFF  ; Emoji_Presentation ; No
-
-231A..231B    ; Emoji_Presentation   #  1.1  [2] (⌚..⌛)    watch..hourglass done
-23E9..23EC    ; Emoji_Presentation   #  6.0  [4] (⏩..⏬)    fast-forward button..fast down button
-23F0          ; Emoji_Presentation   #  6.0  [1] (⏰)       alarm clock
-23F3          ; Emoji_Presentation   #  6.0  [1] (⏳)       hourglass not done
-25FD..25FE    ; Emoji_Presentation   #  3.2  [2] (◽..◾)    white medium-small square..black medium-small square
-2614..2615    ; Emoji_Presentation   #  4.0  [2] (☔..☕)    umbrella with rain drops..hot beverage
-2648..2653    ; Emoji_Presentation   #  1.1 [12] (♈..♓)    Aries..Pisces
-267F          ; Emoji_Presentation   #  4.1  [1] (♿)       wheelchair symbol
-2693          ; Emoji_Presentation   #  4.1  [1] (⚓)       anchor
-26A1          ; Emoji_Presentation   #  4.0  [1] (⚡)       high voltage
-26AA..26AB    ; Emoji_Presentation   #  4.1  [2] (⚪..⚫)    white circle..black circle
-26BD..26BE    ; Emoji_Presentation   #  5.2  [2] (⚽..⚾)    soccer ball..baseball
-26C4..26C5    ; Emoji_Presentation   #  5.2  [2] (⛄..⛅)    snowman without snow..sun behind cloud
-26CE          ; Emoji_Presentation   #  6.0  [1] (⛎)       Ophiuchus
-26D4          ; Emoji_Presentation   #  5.2  [1] (⛔)       no entry
-26EA          ; Emoji_Presentation   #  5.2  [1] (⛪)       church
-26F2..26F3    ; Emoji_Presentation   #  5.2  [2] (⛲..⛳)    fountain..flag in hole
-26F5          ; Emoji_Presentation   #  5.2  [1] (⛵)       sailboat
-26FA          ; Emoji_Presentation   #  5.2  [1] (⛺)       tent
-26FD          ; Emoji_Presentation   #  5.2  [1] (⛽)       fuel pump
-2705          ; Emoji_Presentation   #  6.0  [1] (✅)       check mark button
-270A..270B    ; Emoji_Presentation   #  6.0  [2] (✊..✋)    raised fist..raised hand
-2728          ; Emoji_Presentation   #  6.0  [1] (✨)       sparkles
-274C          ; Emoji_Presentation   #  6.0  [1] (❌)       cross mark
-274E          ; Emoji_Presentation   #  6.0  [1] (❎)       cross mark button
-2753..2755    ; Emoji_Presentation   #  6.0  [3] (❓..❕)    question mark..white exclamation mark
-2757          ; Emoji_Presentation   #  5.2  [1] (❗)       exclamation mark
-2795..2797    ; Emoji_Presentation   #  6.0  [3] (➕..➗)    plus sign..division sign
-27B0          ; Emoji_Presentation   #  6.0  [1] (➰)       curly loop
-27BF          ; Emoji_Presentation   #  6.0  [1] (➿)       double curly loop
-2B1B..2B1C    ; Emoji_Presentation   #  5.1  [2] (⬛..⬜)    black large square..white large square
-2B50          ; Emoji_Presentation   #  5.1  [1] (⭐)       star
-2B55          ; Emoji_Presentation   #  5.2  [1] (⭕)       hollow red circle
-1F004         ; Emoji_Presentation   #  5.1  [1] (🀄)       mahjong red dragon
-1F0CF         ; Emoji_Presentation   #  6.0  [1] (🃏)       joker
-1F18E         ; Emoji_Presentation   #  6.0  [1] (🆎)       AB button (blood type)
-1F191..1F19A  ; Emoji_Presentation   #  6.0 [10] (🆑..🆚)    CL button..VS button
-1F1E6..1F1FF  ; Emoji_Presentation   #  6.0 [26] (🇦..🇿)    regional indicator symbol letter a..regional indicator symbol letter z
-1F201         ; Emoji_Presentation   #  6.0  [1] (🈁)       Japanese “here” button
-1F21A         ; Emoji_Presentation   #  5.2  [1] (🈚)       Japanese “free of charge” button
-1F22F         ; Emoji_Presentation   #  5.2  [1] (🈯)       Japanese “reserved” button
-1F232..1F236  ; Emoji_Presentation   #  6.0  [5] (🈲..🈶)    Japanese “prohibited” button..Japanese “not free of charge” button
-1F238..1F23A  ; Emoji_Presentation   #  6.0  [3] (🈸..🈺)    Japanese “application” button..Japanese “open for business” button
-1F250..1F251  ; Emoji_Presentation   #  6.0  [2] (🉐..🉑)    Japanese “bargain” button..Japanese “acceptable” button
-1F300..1F320  ; Emoji_Presentation   #  6.0 [33] (🌀..🌠)    cyclone..shooting star
-1F32D..1F32F  ; Emoji_Presentation   #  8.0  [3] (🌭..🌯)    hot dog..burrito
-1F330..1F335  ; Emoji_Presentation   #  6.0  [6] (🌰..🌵)    chestnut..cactus
-1F337..1F37C  ; Emoji_Presentation   #  6.0 [70] (🌷..🍼)    tulip..baby bottle
-1F37E..1F37F  ; Emoji_Presentation   #  8.0  [2] (🍾..🍿)    bottle with popping cork..popcorn
-1F380..1F393  ; Emoji_Presentation   #  6.0 [20] (🎀..🎓)    ribbon..graduation cap
-1F3A0..1F3C4  ; Emoji_Presentation   #  6.0 [37] (🎠..🏄)    carousel horse..person surfing
-1F3C5         ; Emoji_Presentation   #  7.0  [1] (🏅)       sports medal
-1F3C6..1F3CA  ; Emoji_Presentation   #  6.0  [5] (🏆..🏊)    trophy..person swimming
-1F3CF..1F3D3  ; Emoji_Presentation   #  8.0  [5] (🏏..🏓)    cricket game..ping pong
-1F3E0..1F3F0  ; Emoji_Presentation   #  6.0 [17] (🏠..🏰)    house..castle
-1F3F4         ; Emoji_Presentation   #  7.0  [1] (🏴)       black flag
-1F3F8..1F3FF  ; Emoji_Presentation   #  8.0  [8] (🏸..🏿)    badminton..dark skin tone
-1F400..1F43E  ; Emoji_Presentation   #  6.0 [63] (🐀..🐾)    rat..paw prints
-1F440         ; Emoji_Presentation   #  6.0  [1] (👀)       eyes
-1F442..1F4F7  ; Emoji_Presentation   #  6.0[182] (👂..📷)    ear..camera
-1F4F8         ; Emoji_Presentation   #  7.0  [1] (📸)       camera with flash
-1F4F9..1F4FC  ; Emoji_Presentation   #  6.0  [4] (📹..📼)    video camera..videocassette
-1F4FF         ; Emoji_Presentation   #  8.0  [1] (📿)       prayer beads
-1F500..1F53D  ; Emoji_Presentation   #  6.0 [62] (🔀..🔽)    shuffle tracks button..downwards button
-1F54B..1F54E  ; Emoji_Presentation   #  8.0  [4] (🕋..🕎)    kaaba..menorah
-1F550..1F567  ; Emoji_Presentation   #  6.0 [24] (🕐..🕧)    one o’clock..twelve-thirty
-1F57A         ; Emoji_Presentation   #  9.0  [1] (🕺)       man dancing
-1F595..1F596  ; Emoji_Presentation   #  7.0  [2] (🖕..🖖)    middle finger..vulcan salute
-1F5A4         ; Emoji_Presentation   #  9.0  [1] (🖤)       black heart
-1F5FB..1F5FF  ; Emoji_Presentation   #  6.0  [5] (🗻..🗿)    mount fuji..moai
-1F600         ; Emoji_Presentation   #  6.1  [1] (😀)       grinning face
-1F601..1F610  ; Emoji_Presentation   #  6.0 [16] (😁..😐)    beaming face with smiling eyes..neutral face
-1F611         ; Emoji_Presentation   #  6.1  [1] (😑)       expressionless face
-1F612..1F614  ; Emoji_Presentation   #  6.0  [3] (😒..😔)    unamused face..pensive face
-1F615         ; Emoji_Presentation   #  6.1  [1] (😕)       confused face
-1F616         ; Emoji_Presentation   #  6.0  [1] (😖)       confounded face
-1F617         ; Emoji_Presentation   #  6.1  [1] (😗)       kissing face
-1F618         ; Emoji_Presentation   #  6.0  [1] (😘)       face blowing a kiss
-1F619         ; Emoji_Presentation   #  6.1  [1] (😙)       kissing face with smiling eyes
-1F61A         ; Emoji_Presentation   #  6.0  [1] (😚)       kissing face with closed eyes
-1F61B         ; Emoji_Presentation   #  6.1  [1] (😛)       face with tongue
-1F61C..1F61E  ; Emoji_Presentation   #  6.0  [3] (😜..😞)    winking face with tongue..disappointed face
-1F61F         ; Emoji_Presentation   #  6.1  [1] (😟)       worried face
-1F620..1F625  ; Emoji_Presentation   #  6.0  [6] (😠..😥)    angry face..sad but relieved face
-1F626..1F627  ; Emoji_Presentation   #  6.1  [2] (😦..😧)    frowning face with open mouth..anguished face
-1F628..1F62B  ; Emoji_Presentation   #  6.0  [4] (😨..😫)    fearful face..tired face
-1F62C         ; Emoji_Presentation   #  6.1  [1] (😬)       grimacing face
-1F62D         ; Emoji_Presentation   #  6.0  [1] (😭)       loudly crying face
-1F62E..1F62F  ; Emoji_Presentation   #  6.1  [2] (😮..😯)    face with open mouth..hushed face
-1F630..1F633  ; Emoji_Presentation   #  6.0  [4] (😰..😳)    anxious face with sweat..flushed face
-1F634         ; Emoji_Presentation   #  6.1  [1] (😴)       sleeping face
-1F635..1F640  ; Emoji_Presentation   #  6.0 [12] (😵..🙀)    dizzy face..weary cat
-1F641..1F642  ; Emoji_Presentation   #  7.0  [2] (🙁..🙂)    slightly frowning face..slightly smiling face
-1F643..1F644  ; Emoji_Presentation   #  8.0  [2] (🙃..🙄)    upside-down face..face with rolling eyes
-1F645..1F64F  ; Emoji_Presentation   #  6.0 [11] (🙅..🙏)    person gesturing NO..folded hands
-1F680..1F6C5  ; Emoji_Presentation   #  6.0 [70] (🚀..🛅)    rocket..left luggage
-1F6CC         ; Emoji_Presentation   #  7.0  [1] (🛌)       person in bed
-1F6D0         ; Emoji_Presentation   #  8.0  [1] (🛐)       place of worship
-1F6D1..1F6D2  ; Emoji_Presentation   #  9.0  [2] (🛑..🛒)    stop sign..shopping cart
-1F6D5         ; Emoji_Presentation   # 12.0  [1] (🛕)       hindu temple
-1F6EB..1F6EC  ; Emoji_Presentation   #  7.0  [2] (🛫..🛬)    airplane departure..airplane arrival
-1F6F4..1F6F6  ; Emoji_Presentation   #  9.0  [3] (🛴..🛶)    kick scooter..canoe
-1F6F7..1F6F8  ; Emoji_Presentation   # 10.0  [2] (🛷..🛸)    sled..flying saucer
-1F6F9         ; Emoji_Presentation   # 11.0  [1] (🛹)       skateboard
-1F6FA         ; Emoji_Presentation   # 12.0  [1] (🛺)       auto rickshaw
-1F7E0..1F7EB  ; Emoji_Presentation   # 12.0 [12] (🟠..🟫)    orange circle..brown square
-1F90D..1F90F  ; Emoji_Presentation   # 12.0  [3] (🤍..🤏)    white heart..pinching hand
-1F910..1F918  ; Emoji_Presentation   #  8.0  [9] (🤐..🤘)    zipper-mouth face..sign of the horns
-1F919..1F91E  ; Emoji_Presentation   #  9.0  [6] (🤙..🤞)    call me hand..crossed fingers
-1F91F         ; Emoji_Presentation   # 10.0  [1] (🤟)       love-you gesture
-1F920..1F927  ; Emoji_Presentation   #  9.0  [8] (🤠..🤧)    cowboy hat face..sneezing face
-1F928..1F92F  ; Emoji_Presentation   # 10.0  [8] (🤨..🤯)    face with raised eyebrow..exploding head
-1F930         ; Emoji_Presentation   #  9.0  [1] (🤰)       pregnant woman
-1F931..1F932  ; Emoji_Presentation   # 10.0  [2] (🤱..🤲)    breast-feeding..palms up together
-1F933..1F93A  ; Emoji_Presentation   #  9.0  [8] (🤳..🤺)    selfie..person fencing
-1F93C..1F93E  ; Emoji_Presentation   #  9.0  [3] (🤼..🤾)    people wrestling..person playing handball
-1F93F         ; Emoji_Presentation   # 12.0  [1] (🤿)       diving mask
-1F940..1F945  ; Emoji_Presentation   #  9.0  [6] (🥀..🥅)    wilted flower..goal net
-1F947..1F94B  ; Emoji_Presentation   #  9.0  [5] (🥇..🥋)    1st place medal..martial arts uniform
-1F94C         ; Emoji_Presentation   # 10.0  [1] (🥌)       curling stone
-1F94D..1F94F  ; Emoji_Presentation   # 11.0  [3] (🥍..🥏)    lacrosse..flying disc
-1F950..1F95E  ; Emoji_Presentation   #  9.0 [15] (🥐..🥞)    croissant..pancakes
-1F95F..1F96B  ; Emoji_Presentation   # 10.0 [13] (🥟..🥫)    dumpling..canned food
-1F96C..1F970  ; Emoji_Presentation   # 11.0  [5] (🥬..🥰)    leafy green..smiling face with hearts
-1F971         ; Emoji_Presentation   # 12.0  [1] (🥱)       yawning face
-1F973..1F976  ; Emoji_Presentation   # 11.0  [4] (🥳..🥶)    partying face..cold face
-1F97A         ; Emoji_Presentation   # 11.0  [1] (🥺)       pleading face
-1F97B         ; Emoji_Presentation   # 12.0  [1] (🥻)       sari
-1F97C..1F97F  ; Emoji_Presentation   # 11.0  [4] (🥼..🥿)    lab coat..flat shoe
-1F980..1F984  ; Emoji_Presentation   #  8.0  [5] (🦀..🦄)    crab..unicorn
-1F985..1F991  ; Emoji_Presentation   #  9.0 [13] (🦅..🦑)    eagle..squid
-1F992..1F997  ; Emoji_Presentation   # 10.0  [6] (🦒..🦗)    giraffe..cricket
-1F998..1F9A2  ; Emoji_Presentation   # 11.0 [11] (🦘..🦢)    kangaroo..swan
-1F9A5..1F9AA  ; Emoji_Presentation   # 12.0  [6] (🦥..🦪)    sloth..oyster
-1F9AE..1F9AF  ; Emoji_Presentation   # 12.0  [2] (🦮..🦯)    guide dog..probing cane
-1F9B0..1F9B9  ; Emoji_Presentation   # 11.0 [10] (🦰..🦹)    red hair..supervillain
-1F9BA..1F9BF  ; Emoji_Presentation   # 12.0  [6] (🦺..🦿)    safety vest..mechanical leg
-1F9C0         ; Emoji_Presentation   #  8.0  [1] (🧀)       cheese wedge
-1F9C1..1F9C2  ; Emoji_Presentation   # 11.0  [2] (🧁..🧂)    cupcake..salt
-1F9C3..1F9CA  ; Emoji_Presentation   # 12.0  [8] (🧃..🧊)    beverage box..ice cube
-1F9CD..1F9CF  ; Emoji_Presentation   # 12.0  [3] (🧍..🧏)    person standing..deaf person
-1F9D0..1F9E6  ; Emoji_Presentation   # 10.0 [23] (🧐..🧦)    face with monocle..socks
-1F9E7..1F9FF  ; Emoji_Presentation   # 11.0 [25] (🧧..🧿)    red envelope..nazar amulet
-1FA70..1FA73  ; Emoji_Presentation   # 12.0  [4] (🩰..🩳)    ballet shoes..shorts
-1FA78..1FA7A  ; Emoji_Presentation   # 12.0  [3] (🩸..🩺)    drop of blood..stethoscope
-1FA80..1FA82  ; Emoji_Presentation   # 12.0  [3] (🪀..🪂)    yo-yo..parachute
-1FA90..1FA95  ; Emoji_Presentation   # 12.0  [6] (🪐..🪕)    ringed planet..banjo
-
-# Total elements: 1093
-
-# ================================================
-
-# All omitted code points have Emoji_Modifier=No 
-# @missing: 0000..10FFFF  ; Emoji_Modifier ; No
-
-1F3FB..1F3FF  ; Emoji_Modifier       #  8.0  [5] (🏻..🏿)    light skin tone..dark skin tone
-
-# Total elements: 5
-
-# ================================================
-
-# All omitted code points have Emoji_Modifier_Base=No 
-# @missing: 0000..10FFFF  ; Emoji_Modifier_Base ; No
-
-261D          ; Emoji_Modifier_Base  #  1.1  [1] (☝️)       index pointing up
-26F9          ; Emoji_Modifier_Base  #  5.2  [1] (⛹️)       person bouncing ball
-270A..270B    ; Emoji_Modifier_Base  #  6.0  [2] (✊..✋)    raised fist..raised hand
-270C..270D    ; Emoji_Modifier_Base  #  1.1  [2] (✌️..✍️)    victory hand..writing hand
-1F385         ; Emoji_Modifier_Base  #  6.0  [1] (🎅)       Santa Claus
-1F3C2..1F3C4  ; Emoji_Modifier_Base  #  6.0  [3] (🏂..🏄)    snowboarder..person surfing
-1F3C7         ; Emoji_Modifier_Base  #  6.0  [1] (🏇)       horse racing
-1F3CA         ; Emoji_Modifier_Base  #  6.0  [1] (🏊)       person swimming
-1F3CB..1F3CC  ; Emoji_Modifier_Base  #  7.0  [2] (🏋️..🏌️)    person lifting weights..person golfing
-1F442..1F443  ; Emoji_Modifier_Base  #  6.0  [2] (👂..👃)    ear..nose
-1F446..1F450  ; Emoji_Modifier_Base  #  6.0 [11] (👆..👐)    backhand index pointing up..open hands
-1F466..1F478  ; Emoji_Modifier_Base  #  6.0 [19] (👦..👸)    boy..princess
-1F47C         ; Emoji_Modifier_Base  #  6.0  [1] (👼)       baby angel
-1F481..1F483  ; Emoji_Modifier_Base  #  6.0  [3] (💁..💃)    person tipping hand..woman dancing
-1F485..1F487  ; Emoji_Modifier_Base  #  6.0  [3] (💅..💇)    nail polish..person getting haircut
-1F48F         ; Emoji_Modifier_Base  #  6.0  [1] (💏)       kiss
-1F491         ; Emoji_Modifier_Base  #  6.0  [1] (💑)       couple with heart
-1F4AA         ; Emoji_Modifier_Base  #  6.0  [1] (💪)       flexed biceps
-1F574..1F575  ; Emoji_Modifier_Base  #  7.0  [2] (🕴️..🕵️)    man in suit levitating..detective
-1F57A         ; Emoji_Modifier_Base  #  9.0  [1] (🕺)       man dancing
-1F590         ; Emoji_Modifier_Base  #  7.0  [1] (🖐️)       hand with fingers splayed
-1F595..1F596  ; Emoji_Modifier_Base  #  7.0  [2] (🖕..🖖)    middle finger..vulcan salute
-1F645..1F647  ; Emoji_Modifier_Base  #  6.0  [3] (🙅..🙇)    person gesturing NO..person bowing
-1F64B..1F64F  ; Emoji_Modifier_Base  #  6.0  [5] (🙋..🙏)    person raising hand..folded hands
-1F6A3         ; Emoji_Modifier_Base  #  6.0  [1] (🚣)       person rowing boat
-1F6B4..1F6B6  ; Emoji_Modifier_Base  #  6.0  [3] (🚴..🚶)    person biking..person walking
-1F6C0         ; Emoji_Modifier_Base  #  6.0  [1] (🛀)       person taking bath
-1F6CC         ; Emoji_Modifier_Base  #  7.0  [1] (🛌)       person in bed
-1F90F         ; Emoji_Modifier_Base  # 12.0  [1] (🤏)       pinching hand
-1F918         ; Emoji_Modifier_Base  #  8.0  [1] (🤘)       sign of the horns
-1F919..1F91E  ; Emoji_Modifier_Base  #  9.0  [6] (🤙..🤞)    call me hand..crossed fingers
-1F91F         ; Emoji_Modifier_Base  # 10.0  [1] (🤟)       love-you gesture
-1F926         ; Emoji_Modifier_Base  #  9.0  [1] (🤦)       person facepalming
-1F930         ; Emoji_Modifier_Base  #  9.0  [1] (🤰)       pregnant woman
-1F931..1F932  ; Emoji_Modifier_Base  # 10.0  [2] (🤱..🤲)    breast-feeding..palms up together
-1F933..1F939  ; Emoji_Modifier_Base  #  9.0  [7] (🤳..🤹)    selfie..person juggling
-1F93C..1F93E  ; Emoji_Modifier_Base  #  9.0  [3] (🤼..🤾)    people wrestling..person playing handball
-1F9B5..1F9B6  ; Emoji_Modifier_Base  # 11.0  [2] (🦵..🦶)    leg..foot
-1F9B8..1F9B9  ; Emoji_Modifier_Base  # 11.0  [2] (🦸..🦹)    superhero..supervillain
-1F9BB         ; Emoji_Modifier_Base  # 12.0  [1] (🦻)       ear with hearing aid
-1F9CD..1F9CF  ; Emoji_Modifier_Base  # 12.0  [3] (🧍..🧏)    person standing..deaf person
-1F9D1..1F9DD  ; Emoji_Modifier_Base  # 10.0 [13] (🧑..🧝)    person..elf
-
-# Total elements: 120
-
-# ================================================
-
-# All omitted code points have Emoji_Component=No 
-# @missing: 0000..10FFFF  ; Emoji_Component ; No
-
-0023          ; Emoji_Component      #  1.1  [1] (#️)       number sign
-002A          ; Emoji_Component      #  1.1  [1] (*️)       asterisk
-0030..0039    ; Emoji_Component      #  1.1 [10] (0️..9️)    digit zero..digit nine
-200D          ; Emoji_Component      #  1.1  [1] (‍)        zero width joiner
-20E3          ; Emoji_Component      #  3.0  [1] (⃣)       combining enclosing keycap
-FE0F          ; Emoji_Component      #  3.2  [1] ()        VARIATION SELECTOR-16
-1F1E6..1F1FF  ; Emoji_Component      #  6.0 [26] (🇦..🇿)    regional indicator symbol letter a..regional indicator symbol letter z
-1F3FB..1F3FF  ; Emoji_Component      #  8.0  [5] (🏻..🏿)    light skin tone..dark skin tone
-1F9B0..1F9B3  ; Emoji_Component      # 11.0  [4] (🦰..🦳)    red hair..white hair
-E0020..E007F  ; Emoji_Component      #  3.1 [96] (󠀠..󠁿)      tag space..cancel tag
-
-# Total elements: 146
-
-# ================================================
-
-# All omitted code points have Extended_Pictographic=No 
-# @missing: 0000..10FFFF  ; Extended_Pictographic ; No
-
-00A9          ; Extended_Pictographic#  1.1  [1] (©️)       copyright
-00AE          ; Extended_Pictographic#  1.1  [1] (®️)       registered
-203C          ; Extended_Pictographic#  1.1  [1] (‼️)       double exclamation mark
-2049          ; Extended_Pictographic#  3.0  [1] (⁉️)       exclamation question mark
-2122          ; Extended_Pictographic#  1.1  [1] (™️)       trade mark
-2139          ; Extended_Pictographic#  3.0  [1] (ℹ️)       information
-2194..2199    ; Extended_Pictographic#  1.1  [6] (↔️..↙️)    left-right arrow..down-left arrow
-21A9..21AA    ; Extended_Pictographic#  1.1  [2] (↩️..↪️)    right arrow curving left..left arrow curving right
-231A..231B    ; Extended_Pictographic#  1.1  [2] (⌚..⌛)    watch..hourglass done
-2328          ; Extended_Pictographic#  1.1  [1] (⌨️)       keyboard
-2388          ; Extended_Pictographic#  3.0  [1] (⎈)       HELM SYMBOL
-23CF          ; Extended_Pictographic#  4.0  [1] (⏏️)       eject button
-23E9..23F3    ; Extended_Pictographic#  6.0 [11] (⏩..⏳)    fast-forward button..hourglass not done
-23F8..23FA    ; Extended_Pictographic#  7.0  [3] (⏸️..⏺️)    pause button..record button
-24C2          ; Extended_Pictographic#  1.1  [1] (Ⓜ️)       circled M
-25AA..25AB    ; Extended_Pictographic#  1.1  [2] (▪️..▫️)    black small square..white small square
-25B6          ; Extended_Pictographic#  1.1  [1] (▶️)       play button
-25C0          ; Extended_Pictographic#  1.1  [1] (◀️)       reverse button
-25FB..25FE    ; Extended_Pictographic#  3.2  [4] (◻️..◾)    white medium square..black medium-small square
-2600..2605    ; Extended_Pictographic#  1.1  [6] (☀️..★)    sun..BLACK STAR
-2607..2612    ; Extended_Pictographic#  1.1 [12] (☇..☒)    LIGHTNING..BALLOT BOX WITH X
-2614..2615    ; Extended_Pictographic#  4.0  [2] (☔..☕)    umbrella with rain drops..hot beverage
-2616..2617    ; Extended_Pictographic#  3.2  [2] (☖..☗)    WHITE SHOGI PIECE..BLACK SHOGI PIECE
-2618          ; Extended_Pictographic#  4.1  [1] (☘️)       shamrock
-2619          ; Extended_Pictographic#  3.0  [1] (☙)       REVERSED ROTATED FLORAL HEART BULLET
-261A..266F    ; Extended_Pictographic#  1.1 [86] (☚..♯)    BLACK LEFT POINTING INDEX..MUSIC SHARP SIGN
-2670..2671    ; Extended_Pictographic#  3.0  [2] (♰..♱)    WEST SYRIAC CROSS..EAST SYRIAC CROSS
-2672..267D    ; Extended_Pictographic#  3.2 [12] (♲..♽)    UNIVERSAL RECYCLING SYMBOL..PARTIALLY-RECYCLED PAPER SYMBOL
-267E..267F    ; Extended_Pictographic#  4.1  [2] (♾️..♿)    infinity..wheelchair symbol
-2680..2685    ; Extended_Pictographic#  3.2  [6] (⚀..⚅)    DIE FACE-1..DIE FACE-6
-2690..2691    ; Extended_Pictographic#  4.0  [2] (⚐..⚑)    WHITE FLAG..BLACK FLAG
-2692..269C    ; Extended_Pictographic#  4.1 [11] (⚒️..⚜️)    hammer and pick..fleur-de-lis
-269D          ; Extended_Pictographic#  5.1  [1] (⚝)       OUTLINED WHITE STAR
-269E..269F    ; Extended_Pictographic#  5.2  [2] (⚞..⚟)    THREE LINES CONVERGING RIGHT..THREE LINES CONVERGING LEFT
-26A0..26A1    ; Extended_Pictographic#  4.0  [2] (⚠️..⚡)    warning..high voltage
-26A2..26B1    ; Extended_Pictographic#  4.1 [16] (⚢..⚱️)    DOUBLED FEMALE SIGN..funeral urn
-26B2          ; Extended_Pictographic#  5.0  [1] (⚲)       NEUTER
-26B3..26BC    ; Extended_Pictographic#  5.1 [10] (⚳..⚼)    CERES..SESQUIQUADRATE
-26BD..26BF    ; Extended_Pictographic#  5.2  [3] (⚽..⚿)    soccer ball..SQUARED KEY
-26C0..26C3    ; Extended_Pictographic#  5.1  [4] (⛀..⛃)    WHITE DRAUGHTS MAN..BLACK DRAUGHTS KING
-26C4..26CD    ; Extended_Pictographic#  5.2 [10] (⛄..⛍)    snowman without snow..DISABLED CAR
-26CE          ; Extended_Pictographic#  6.0  [1] (⛎)       Ophiuchus
-26CF..26E1    ; Extended_Pictographic#  5.2 [19] (⛏️..⛡)    pick..RESTRICTED LEFT ENTRY-2
-26E2          ; Extended_Pictographic#  6.0  [1] (⛢)       ASTRONOMICAL SYMBOL FOR URANUS
-26E3          ; Extended_Pictographic#  5.2  [1] (⛣)       HEAVY CIRCLE WITH STROKE AND TWO DOTS ABOVE
-26E4..26E7    ; Extended_Pictographic#  6.0  [4] (⛤..⛧)    PENTAGRAM..INVERTED PENTAGRAM
-26E8..26FF    ; Extended_Pictographic#  5.2 [24] (⛨..⛿)    BLACK CROSS ON SHIELD..WHITE FLAG WITH HORIZONTAL MIDDLE BLACK STRIPE
-2700          ; Extended_Pictographic#  7.0  [1] (✀)       BLACK SAFETY SCISSORS
-2701..2704    ; Extended_Pictographic#  1.1  [4] (✁..✄)    UPPER BLADE SCISSORS..WHITE SCISSORS
-2705          ; Extended_Pictographic#  6.0  [1] (✅)       check mark button
-2708..2709    ; Extended_Pictographic#  1.1  [2] (✈️..✉️)    airplane..envelope
-270A..270B    ; Extended_Pictographic#  6.0  [2] (✊..✋)    raised fist..raised hand
-270C..2712    ; Extended_Pictographic#  1.1  [7] (✌️..✒️)    victory hand..black nib
-2714          ; Extended_Pictographic#  1.1  [1] (✔️)       check mark
-2716          ; Extended_Pictographic#  1.1  [1] (✖️)       multiplication sign
-271D          ; Extended_Pictographic#  1.1  [1] (✝️)       latin cross
-2721          ; Extended_Pictographic#  1.1  [1] (✡️)       star of David
-2728          ; Extended_Pictographic#  6.0  [1] (✨)       sparkles
-2733..2734    ; Extended_Pictographic#  1.1  [2] (✳️..✴️)    eight-spoked asterisk..eight-pointed star
-2744          ; Extended_Pictographic#  1.1  [1] (❄️)       snowflake
-2747          ; Extended_Pictographic#  1.1  [1] (❇️)       sparkle
-274C          ; Extended_Pictographic#  6.0  [1] (❌)       cross mark
-274E          ; Extended_Pictographic#  6.0  [1] (❎)       cross mark button
-2753..2755    ; Extended_Pictographic#  6.0  [3] (❓..❕)    question mark..white exclamation mark
-2757          ; Extended_Pictographic#  5.2  [1] (❗)       exclamation mark
-2763..2767    ; Extended_Pictographic#  1.1  [5] (❣️..❧)    heart exclamation..ROTATED FLORAL HEART BULLET
-2795..2797    ; Extended_Pictographic#  6.0  [3] (➕..➗)    plus sign..division sign
-27A1          ; Extended_Pictographic#  1.1  [1] (➡️)       right arrow
-27B0          ; Extended_Pictographic#  6.0  [1] (➰)       curly loop
-27BF          ; Extended_Pictographic#  6.0  [1] (➿)       double curly loop
-2934..2935    ; Extended_Pictographic#  3.2  [2] (⤴️..⤵️)    right arrow curving up..right arrow curving down
-2B05..2B07    ; Extended_Pictographic#  4.0  [3] (⬅️..⬇️)    left arrow..down arrow
-2B1B..2B1C    ; Extended_Pictographic#  5.1  [2] (⬛..⬜)    black large square..white large square
-2B50          ; Extended_Pictographic#  5.1  [1] (⭐)       star
-2B55          ; Extended_Pictographic#  5.2  [1] (⭕)       hollow red circle
-3030          ; Extended_Pictographic#  1.1  [1] (〰️)       wavy dash
-303D          ; Extended_Pictographic#  3.2  [1] (〽️)       part alternation mark
-3297          ; Extended_Pictographic#  1.1  [1] (㊗️)       Japanese “congratulations” button
-3299          ; Extended_Pictographic#  1.1  [1] (㊙️)       Japanese “secret” button
-1F000..1F02B  ; Extended_Pictographic#  5.1 [44] (🀀..🀫)    MAHJONG TILE EAST WIND..MAHJONG TILE BACK
-1F02C..1F02F  ; Extended_Pictographic#   NA  [4] (🀬..🀯)    <reserved-1F02C>..<reserved-1F02F>
-1F030..1F093  ; Extended_Pictographic#  5.1[100] (🀰..🂓)    DOMINO TILE HORIZONTAL BACK..DOMINO TILE VERTICAL-06-06
-1F094..1F09F  ; Extended_Pictographic#   NA [12] (🂔..🂟)    <reserved-1F094>..<reserved-1F09F>
-1F0A0..1F0AE  ; Extended_Pictographic#  6.0 [15] (🂠..🂮)    PLAYING CARD BACK..PLAYING CARD KING OF SPADES
-1F0AF..1F0B0  ; Extended_Pictographic#   NA  [2] (🂯..🂰)    <reserved-1F0AF>..<reserved-1F0B0>
-1F0B1..1F0BE  ; Extended_Pictographic#  6.0 [14] (🂱..🂾)    PLAYING CARD ACE OF HEARTS..PLAYING CARD KING OF HEARTS
-1F0BF         ; Extended_Pictographic#  7.0  [1] (🂿)       PLAYING CARD RED JOKER
-1F0C0         ; Extended_Pictographic#   NA  [1] (🃀)       <reserved-1F0C0>
-1F0C1..1F0CF  ; Extended_Pictographic#  6.0 [15] (🃁..🃏)    PLAYING CARD ACE OF DIAMONDS..joker
-1F0D0         ; Extended_Pictographic#   NA  [1] (🃐)       <reserved-1F0D0>
-1F0D1..1F0DF  ; Extended_Pictographic#  6.0 [15] (🃑..🃟)    PLAYING CARD ACE OF CLUBS..PLAYING CARD WHITE JOKER
-1F0E0..1F0F5  ; Extended_Pictographic#  7.0 [22] (🃠..🃵)    PLAYING CARD FOOL..PLAYING CARD TRUMP-21
-1F0F6..1F0FF  ; Extended_Pictographic#   NA [10] (🃶..🃿)    <reserved-1F0F6>..<reserved-1F0FF>
-1F10D..1F10F  ; Extended_Pictographic#   NA  [3] (🄍..🄏)    <reserved-1F10D>..<reserved-1F10F>
-1F12F         ; Extended_Pictographic# 11.0  [1] (🄯)       COPYLEFT SYMBOL
-1F16C         ; Extended_Pictographic# 12.0  [1] (🅬)       RAISED MR SIGN
-1F16D..1F16F  ; Extended_Pictographic#   NA  [3] (🅭..🅯)    <reserved-1F16D>..<reserved-1F16F>
-1F170..1F171  ; Extended_Pictographic#  6.0  [2] (🅰️..🅱️)    A button (blood type)..B button (blood type)
-1F17E         ; Extended_Pictographic#  6.0  [1] (🅾️)       O button (blood type)
-1F17F         ; Extended_Pictographic#  5.2  [1] (🅿️)       P button
-1F18E         ; Extended_Pictographic#  6.0  [1] (🆎)       AB button (blood type)
-1F191..1F19A  ; Extended_Pictographic#  6.0 [10] (🆑..🆚)    CL button..VS button
-1F1AD..1F1E5  ; Extended_Pictographic#   NA [57] (🆭..🇥)    <reserved-1F1AD>..<reserved-1F1E5>
-1F201..1F202  ; Extended_Pictographic#  6.0  [2] (🈁..🈂️)    Japanese “here” button..Japanese “service charge” button
-1F203..1F20F  ; Extended_Pictographic#   NA [13] (🈃..🈏)    <reserved-1F203>..<reserved-1F20F>
-1F21A         ; Extended_Pictographic#  5.2  [1] (🈚)       Japanese “free of charge” button
-1F22F         ; Extended_Pictographic#  5.2  [1] (🈯)       Japanese “reserved” button
-1F232..1F23A  ; Extended_Pictographic#  6.0  [9] (🈲..🈺)    Japanese “prohibited” button..Japanese “open for business” button
-1F23C..1F23F  ; Extended_Pictographic#   NA  [4] (🈼..🈿)    <reserved-1F23C>..<reserved-1F23F>
-1F249..1F24F  ; Extended_Pictographic#   NA  [7] (🉉..🉏)    <reserved-1F249>..<reserved-1F24F>
-1F250..1F251  ; Extended_Pictographic#  6.0  [2] (🉐..🉑)    Japanese “bargain” button..Japanese “acceptable” button
-1F252..1F25F  ; Extended_Pictographic#   NA [14] (🉒..🉟)    <reserved-1F252>..<reserved-1F25F>
-1F260..1F265  ; Extended_Pictographic# 10.0  [6] (🉠..🉥)    ROUNDED SYMBOL FOR FU..ROUNDED SYMBOL FOR CAI
-1F266..1F2FF  ; Extended_Pictographic#   NA[154] (🉦..🋿)    <reserved-1F266>..<reserved-1F2FF>
-1F300..1F320  ; Extended_Pictographic#  6.0 [33] (🌀..🌠)    cyclone..shooting star
-1F321..1F32C  ; Extended_Pictographic#  7.0 [12] (🌡️..🌬️)    thermometer..wind face
-1F32D..1F32F  ; Extended_Pictographic#  8.0  [3] (🌭..🌯)    hot dog..burrito
-1F330..1F335  ; Extended_Pictographic#  6.0  [6] (🌰..🌵)    chestnut..cactus
-1F336         ; Extended_Pictographic#  7.0  [1] (🌶️)       hot pepper
-1F337..1F37C  ; Extended_Pictographic#  6.0 [70] (🌷..🍼)    tulip..baby bottle
-1F37D         ; Extended_Pictographic#  7.0  [1] (🍽️)       fork and knife with plate
-1F37E..1F37F  ; Extended_Pictographic#  8.0  [2] (🍾..🍿)    bottle with popping cork..popcorn
-1F380..1F393  ; Extended_Pictographic#  6.0 [20] (🎀..🎓)    ribbon..graduation cap
-1F394..1F39F  ; Extended_Pictographic#  7.0 [12] (🎔..🎟️)    HEART WITH TIP ON THE LEFT..admission tickets
-1F3A0..1F3C4  ; Extended_Pictographic#  6.0 [37] (🎠..🏄)    carousel horse..person surfing
-1F3C5         ; Extended_Pictographic#  7.0  [1] (🏅)       sports medal
-1F3C6..1F3CA  ; Extended_Pictographic#  6.0  [5] (🏆..🏊)    trophy..person swimming
-1F3CB..1F3CE  ; Extended_Pictographic#  7.0  [4] (🏋️..🏎️)    person lifting weights..racing car
-1F3CF..1F3D3  ; Extended_Pictographic#  8.0  [5] (🏏..🏓)    cricket game..ping pong
-1F3D4..1F3DF  ; Extended_Pictographic#  7.0 [12] (🏔️..🏟️)    snow-capped mountain..stadium
-1F3E0..1F3F0  ; Extended_Pictographic#  6.0 [17] (🏠..🏰)    house..castle
-1F3F1..1F3F7  ; Extended_Pictographic#  7.0  [7] (🏱..🏷️)    WHITE PENNANT..label
-1F3F8..1F3FA  ; Extended_Pictographic#  8.0  [3] (🏸..🏺)    badminton..amphora
-1F400..1F43E  ; Extended_Pictographic#  6.0 [63] (🐀..🐾)    rat..paw prints
-1F43F         ; Extended_Pictographic#  7.0  [1] (🐿️)       chipmunk
-1F440         ; Extended_Pictographic#  6.0  [1] (👀)       eyes
-1F441         ; Extended_Pictographic#  7.0  [1] (👁️)       eye
-1F442..1F4F7  ; Extended_Pictographic#  6.0[182] (👂..📷)    ear..camera
-1F4F8         ; Extended_Pictographic#  7.0  [1] (📸)       camera with flash
-1F4F9..1F4FC  ; Extended_Pictographic#  6.0  [4] (📹..📼)    video camera..videocassette
-1F4FD..1F4FE  ; Extended_Pictographic#  7.0  [2] (📽️..📾)    film projector..PORTABLE STEREO
-1F4FF         ; Extended_Pictographic#  8.0  [1] (📿)       prayer beads
-1F500..1F53D  ; Extended_Pictographic#  6.0 [62] (🔀..🔽)    shuffle tracks button..downwards button
-1F546..1F54A  ; Extended_Pictographic#  7.0  [5] (🕆..🕊️)    WHITE LATIN CROSS..dove
-1F54B..1F54F  ; Extended_Pictographic#  8.0  [5] (🕋..🕏)    kaaba..BOWL OF HYGIEIA
-1F550..1F567  ; Extended_Pictographic#  6.0 [24] (🕐..🕧)    one o’clock..twelve-thirty
-1F568..1F579  ; Extended_Pictographic#  7.0 [18] (🕨..🕹️)    RIGHT SPEAKER..joystick
-1F57A         ; Extended_Pictographic#  9.0  [1] (🕺)       man dancing
-1F57B..1F5A3  ; Extended_Pictographic#  7.0 [41] (🕻..🖣)    LEFT HAND TELEPHONE RECEIVER..BLACK DOWN POINTING BACKHAND INDEX
-1F5A4         ; Extended_Pictographic#  9.0  [1] (🖤)       black heart
-1F5A5..1F5FA  ; Extended_Pictographic#  7.0 [86] (🖥️..🗺️)    desktop computer..world map
-1F5FB..1F5FF  ; Extended_Pictographic#  6.0  [5] (🗻..🗿)    mount fuji..moai
-1F600         ; Extended_Pictographic#  6.1  [1] (😀)       grinning face
-1F601..1F610  ; Extended_Pictographic#  6.0 [16] (😁..😐)    beaming face with smiling eyes..neutral face
-1F611         ; Extended_Pictographic#  6.1  [1] (😑)       expressionless face
-1F612..1F614  ; Extended_Pictographic#  6.0  [3] (😒..😔)    unamused face..pensive face
-1F615         ; Extended_Pictographic#  6.1  [1] (😕)       confused face
-1F616         ; Extended_Pictographic#  6.0  [1] (😖)       confounded face
-1F617         ; Extended_Pictographic#  6.1  [1] (😗)       kissing face
-1F618         ; Extended_Pictographic#  6.0  [1] (😘)       face blowing a kiss
-1F619         ; Extended_Pictographic#  6.1  [1] (😙)       kissing face with smiling eyes
-1F61A         ; Extended_Pictographic#  6.0  [1] (😚)       kissing face with closed eyes
-1F61B         ; Extended_Pictographic#  6.1  [1] (😛)       face with tongue
-1F61C..1F61E  ; Extended_Pictographic#  6.0  [3] (😜..😞)    winking face with tongue..disappointed face
-1F61F         ; Extended_Pictographic#  6.1  [1] (😟)       worried face
-1F620..1F625  ; Extended_Pictographic#  6.0  [6] (😠..😥)    angry face..sad but relieved face
-1F626..1F627  ; Extended_Pictographic#  6.1  [2] (😦..😧)    frowning face with open mouth..anguished face
-1F628..1F62B  ; Extended_Pictographic#  6.0  [4] (😨..😫)    fearful face..tired face
-1F62C         ; Extended_Pictographic#  6.1  [1] (😬)       grimacing face
-1F62D         ; Extended_Pictographic#  6.0  [1] (😭)       loudly crying face
-1F62E..1F62F  ; Extended_Pictographic#  6.1  [2] (😮..😯)    face with open mouth..hushed face
-1F630..1F633  ; Extended_Pictographic#  6.0  [4] (😰..😳)    anxious face with sweat..flushed face
-1F634         ; Extended_Pictographic#  6.1  [1] (😴)       sleeping face
-1F635..1F640  ; Extended_Pictographic#  6.0 [12] (😵..🙀)    dizzy face..weary cat
-1F641..1F642  ; Extended_Pictographic#  7.0  [2] (🙁..🙂)    slightly frowning face..slightly smiling face
-1F643..1F644  ; Extended_Pictographic#  8.0  [2] (🙃..🙄)    upside-down face..face with rolling eyes
-1F645..1F64F  ; Extended_Pictographic#  6.0 [11] (🙅..🙏)    person gesturing NO..folded hands
-1F680..1F6C5  ; Extended_Pictographic#  6.0 [70] (🚀..🛅)    rocket..left luggage
-1F6C6..1F6CF  ; Extended_Pictographic#  7.0 [10] (🛆..🛏️)    TRIANGLE WITH ROUNDED CORNERS..bed
-1F6D0         ; Extended_Pictographic#  8.0  [1] (🛐)       place of worship
-1F6D1..1F6D2  ; Extended_Pictographic#  9.0  [2] (🛑..🛒)    stop sign..shopping cart
-1F6D3..1F6D4  ; Extended_Pictographic# 10.0  [2] (🛓..🛔)    STUPA..PAGODA
-1F6D5         ; Extended_Pictographic# 12.0  [1] (🛕)       hindu temple
-1F6D6..1F6DF  ; Extended_Pictographic#   NA [10] (🛖..🛟)    <reserved-1F6D6>..<reserved-1F6DF>
-1F6E0..1F6EC  ; Extended_Pictographic#  7.0 [13] (🛠️..🛬)    hammer and wrench..airplane arrival
-1F6ED..1F6EF  ; Extended_Pictographic#   NA  [3] (🛭..🛯)    <reserved-1F6ED>..<reserved-1F6EF>
-1F6F0..1F6F3  ; Extended_Pictographic#  7.0  [4] (🛰️..🛳️)    satellite..passenger ship
-1F6F4..1F6F6  ; Extended_Pictographic#  9.0  [3] (🛴..🛶)    kick scooter..canoe
-1F6F7..1F6F8  ; Extended_Pictographic# 10.0  [2] (🛷..🛸)    sled..flying saucer
-1F6F9         ; Extended_Pictographic# 11.0  [1] (🛹)       skateboard
-1F6FA         ; Extended_Pictographic# 12.0  [1] (🛺)       auto rickshaw
-1F6FB..1F6FF  ; Extended_Pictographic#   NA  [5] (🛻..🛿)    <reserved-1F6FB>..<reserved-1F6FF>
-1F774..1F77F  ; Extended_Pictographic#   NA [12] (🝴..🝿)    <reserved-1F774>..<reserved-1F77F>
-1F7D5..1F7D8  ; Extended_Pictographic# 11.0  [4] (🟕..🟘)    CIRCLED TRIANGLE..NEGATIVE CIRCLED SQUARE
-1F7D9..1F7DF  ; Extended_Pictographic#   NA  [7] (🟙..🟟)    <reserved-1F7D9>..<reserved-1F7DF>
-1F7E0..1F7EB  ; Extended_Pictographic# 12.0 [12] (🟠..🟫)    orange circle..brown square
-1F7EC..1F7FF  ; Extended_Pictographic#   NA [20] (🟬..🟿)    <reserved-1F7EC>..<reserved-1F7FF>
-1F80C..1F80F  ; Extended_Pictographic#   NA  [4] (🠌..🠏)    <reserved-1F80C>..<reserved-1F80F>
-1F848..1F84F  ; Extended_Pictographic#   NA  [8] (🡈..🡏)    <reserved-1F848>..<reserved-1F84F>
-1F85A..1F85F  ; Extended_Pictographic#   NA  [6] (🡚..🡟)    <reserved-1F85A>..<reserved-1F85F>
-1F888..1F88F  ; Extended_Pictographic#   NA  [8] (🢈..🢏)    <reserved-1F888>..<reserved-1F88F>
-1F8AE..1F8FF  ; Extended_Pictographic#   NA [82] (🢮..🣿)    <reserved-1F8AE>..<reserved-1F8FF>
-1F90C         ; Extended_Pictographic#   NA  [1] (🤌)       <reserved-1F90C>
-1F90D..1F90F  ; Extended_Pictographic# 12.0  [3] (🤍..🤏)    white heart..pinching hand
-1F910..1F918  ; Extended_Pictographic#  8.0  [9] (🤐..🤘)    zipper-mouth face..sign of the horns
-1F919..1F91E  ; Extended_Pictographic#  9.0  [6] (🤙..🤞)    call me hand..crossed fingers
-1F91F         ; Extended_Pictographic# 10.0  [1] (🤟)       love-you gesture
-1F920..1F927  ; Extended_Pictographic#  9.0  [8] (🤠..🤧)    cowboy hat face..sneezing face
-1F928..1F92F  ; Extended_Pictographic# 10.0  [8] (🤨..🤯)    face with raised eyebrow..exploding head
-1F930         ; Extended_Pictographic#  9.0  [1] (🤰)       pregnant woman
-1F931..1F932  ; Extended_Pictographic# 10.0  [2] (🤱..🤲)    breast-feeding..palms up together
-1F933..1F93A  ; Extended_Pictographic#  9.0  [8] (🤳..🤺)    selfie..person fencing
-1F93C..1F93E  ; Extended_Pictographic#  9.0  [3] (🤼..🤾)    people wrestling..person playing handball
-1F93F         ; Extended_Pictographic# 12.0  [1] (🤿)       diving mask
-1F940..1F945  ; Extended_Pictographic#  9.0  [6] (🥀..🥅)    wilted flower..goal net
-1F947..1F94B  ; Extended_Pictographic#  9.0  [5] (🥇..🥋)    1st place medal..martial arts uniform
-1F94C         ; Extended_Pictographic# 10.0  [1] (🥌)       curling stone
-1F94D..1F94F  ; Extended_Pictographic# 11.0  [3] (🥍..🥏)    lacrosse..flying disc
-1F950..1F95E  ; Extended_Pictographic#  9.0 [15] (🥐..🥞)    croissant..pancakes
-1F95F..1F96B  ; Extended_Pictographic# 10.0 [13] (🥟..🥫)    dumpling..canned food
-1F96C..1F970  ; Extended_Pictographic# 11.0  [5] (🥬..🥰)    leafy green..smiling face with hearts
-1F971         ; Extended_Pictographic# 12.0  [1] (🥱)       yawning face
-1F972         ; Extended_Pictographic#   NA  [1] (🥲)       <reserved-1F972>
-1F973..1F976  ; Extended_Pictographic# 11.0  [4] (🥳..🥶)    partying face..cold face
-1F977..1F979  ; Extended_Pictographic#   NA  [3] (🥷..🥹)    <reserved-1F977>..<reserved-1F979>
-1F97A         ; Extended_Pictographic# 11.0  [1] (🥺)       pleading face
-1F97B         ; Extended_Pictographic# 12.0  [1] (🥻)       sari
-1F97C..1F97F  ; Extended_Pictographic# 11.0  [4] (🥼..🥿)    lab coat..flat shoe
-1F980..1F984  ; Extended_Pictographic#  8.0  [5] (🦀..🦄)    crab..unicorn
-1F985..1F991  ; Extended_Pictographic#  9.0 [13] (🦅..🦑)    eagle..squid
-1F992..1F997  ; Extended_Pictographic# 10.0  [6] (🦒..🦗)    giraffe..cricket
-1F998..1F9A2  ; Extended_Pictographic# 11.0 [11] (🦘..🦢)    kangaroo..swan
-1F9A3..1F9A4  ; Extended_Pictographic#   NA  [2] (🦣..🦤)    <reserved-1F9A3>..<reserved-1F9A4>
-1F9A5..1F9AA  ; Extended_Pictographic# 12.0  [6] (🦥..🦪)    sloth..oyster
-1F9AB..1F9AD  ; Extended_Pictographic#   NA  [3] (🦫..🦭)    <reserved-1F9AB>..<reserved-1F9AD>
-1F9AE..1F9AF  ; Extended_Pictographic# 12.0  [2] (🦮..🦯)    guide dog..probing cane
-1F9B0..1F9B9  ; Extended_Pictographic# 11.0 [10] (🦰..🦹)    red hair..supervillain
-1F9BA..1F9BF  ; Extended_Pictographic# 12.0  [6] (🦺..🦿)    safety vest..mechanical leg
-1F9C0         ; Extended_Pictographic#  8.0  [1] (🧀)       cheese wedge
-1F9C1..1F9C2  ; Extended_Pictographic# 11.0  [2] (🧁..🧂)    cupcake..salt
-1F9C3..1F9CA  ; Extended_Pictographic# 12.0  [8] (🧃..🧊)    beverage box..ice cube
-1F9CB..1F9CC  ; Extended_Pictographic#   NA  [2] (🧋..🧌)    <reserved-1F9CB>..<reserved-1F9CC>
-1F9CD..1F9CF  ; Extended_Pictographic# 12.0  [3] (🧍..🧏)    person standing..deaf person
-1F9D0..1F9E6  ; Extended_Pictographic# 10.0 [23] (🧐..🧦)    face with monocle..socks
-1F9E7..1F9FF  ; Extended_Pictographic# 11.0 [25] (🧧..🧿)    red envelope..nazar amulet
-1FA00..1FA53  ; Extended_Pictographic# 12.0 [84] (🨀..🩓)    NEUTRAL CHESS KING..BLACK CHESS KNIGHT-BISHOP
-1FA54..1FA5F  ; Extended_Pictographic#   NA [12] (🩔..🩟)    <reserved-1FA54>..<reserved-1FA5F>
-1FA60..1FA6D  ; Extended_Pictographic# 11.0 [14] (🩠..🩭)    XIANGQI RED GENERAL..XIANGQI BLACK SOLDIER
-1FA6E..1FA6F  ; Extended_Pictographic#   NA  [2] (🩮..🩯)    <reserved-1FA6E>..<reserved-1FA6F>
-1FA70..1FA73  ; Extended_Pictographic# 12.0  [4] (🩰..🩳)    ballet shoes..shorts
-1FA74..1FA77  ; Extended_Pictographic#   NA  [4] (🩴..🩷)    <reserved-1FA74>..<reserved-1FA77>
-1FA78..1FA7A  ; Extended_Pictographic# 12.0  [3] (🩸..🩺)    drop of blood..stethoscope
-1FA7B..1FA7F  ; Extended_Pictographic#   NA  [5] (🩻..🩿)    <reserved-1FA7B>..<reserved-1FA7F>
-1FA80..1FA82  ; Extended_Pictographic# 12.0  [3] (🪀..🪂)    yo-yo..parachute
-1FA83..1FA8F  ; Extended_Pictographic#   NA [13] (🪃..🪏)    <reserved-1FA83>..<reserved-1FA8F>
-1FA90..1FA95  ; Extended_Pictographic# 12.0  [6] (🪐..🪕)    ringed planet..banjo
-1FA96..1FFFD  ; Extended_Pictographic#   NA[1384] (🪖..🿽)   <reserved-1FA96>..<reserved-1FFFD>
-
-# Total elements: 3793
-
-#EOF

lib/pleroma/emoji.ex

@@ -102,31 +102,36 @@ defmodule Pleroma.Emoji do
     :ets.insert(@ets, emojis)
   end
 
-  @external_resource "lib/pleroma/emoji-data.txt"
+  @external_resource "lib/pleroma/emoji-test.txt"
+
+  regional_indicators =
+    Enum.map(127_462..127_487, fn codepoint ->
+      <<codepoint::utf8>>
+    end)
 
   emojis =
     @external_resource
     |> File.read!()
     |> String.split("\n")
-    |> Enum.filter(fn line -> line != "" and not String.starts_with?(line, "#") end)
+    |> Enum.filter(fn line ->
+      line != "" and not String.starts_with?(line, "#") and
+        String.contains?(line, "fully-qualified")
+    end)
     |> Enum.map(fn line ->
       line
       |> String.split(";", parts: 2)
       |> hd()
       |> String.trim()
-      |> String.split("..")
+      |> String.split()
-      |> case do
+      |> Enum.map(fn codepoint ->
-        [number] ->
+        <<String.to_integer(codepoint, 16)::utf8>>
-          <<String.to_integer(number, 16)::utf8>>
+      end)
-
+      |> Enum.join()
-        [first, last] ->
-          String.to_integer(first, 16)..String.to_integer(last, 16)
-          |> Enum.map(&<<&1::utf8>>)
-      end
     end)
-    |> List.flatten()
     |> Enum.uniq()
 
+  emojis = emojis ++ regional_indicators
+
   for emoji <- emojis do
     def is_unicode_emoji?(unquote(emoji)), do: true
   end

lib/pleroma/emoji/pack.ex

@@ -20,16 +20,18 @@ defmodule Pleroma.Emoji.Pack do
           name: String.t()
         }
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
   alias Pleroma.Emoji
   alias Pleroma.Emoji.Pack
+  alias Pleroma.Utils
 
   @spec create(String.t()) :: {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
   def create(name) do
     with :ok <- validate_not_empty([name]),
          dir <- Path.join(emoji_path(), name),
          :ok <- File.mkdir(dir) do
-      %__MODULE__{pack_file: Path.join(dir, "pack.json")}
+      save_pack(%__MODULE__{pack_file: Path.join(dir, "pack.json")})
-      |> save_pack()
     end
   end
 
@@ -62,10 +64,9 @@ defmodule Pleroma.Emoji.Pack do
   @spec delete(String.t()) ::
           {:ok, [binary()]} | {:error, File.posix(), binary()} | {:error, :empty_values}
   def delete(name) do
-    with :ok <- validate_not_empty([name]) do
+    with :ok <- validate_not_empty([name]),
-      emoji_path()
+         pack_path <- Path.join(emoji_path(), name) do
-      |> Path.join(name)
+      File.rm_rf(pack_path)
-      |> File.rm_rf()
     end
   end
 
@@ -94,7 +95,7 @@ defmodule Pleroma.Emoji.Pack do
   def add_file(%Pack{} = pack, _, _, %Plug.Upload{content_type: "application/zip"} = file) do
     with {:ok, zip_files} <- :zip.table(to_charlist(file.path)),
          [_ | _] = emojies <- unpack_zip_emojies(zip_files),
-         {:ok, tmp_dir} <- Pleroma.Utils.tmp_dir("emoji") do
+         {:ok, tmp_dir} <- Utils.tmp_dir("emoji") do
       try do
         {:ok, _emoji_files} =
           :zip.unzip(
@@ -282,18 +283,21 @@ defmodule Pleroma.Emoji.Pack do
     end
   end
 
-  @spec load_pack(String.t()) :: {:ok, t()} | {:error, :not_found}
+  @spec load_pack(String.t()) :: {:ok, t()} | {:error, :file.posix()}
   def load_pack(name) do
     pack_file = Path.join([emoji_path(), name, "pack.json"])
 
-    if File.exists?(pack_file) do
+    with {:ok, _} <- File.stat(pack_file),
+         {:ok, pack_data} <- File.read(pack_file) do
       pack =
-        pack_file
+        from_json(
-        |> File.read!()
+          pack_data,
-        |> from_json()
+          %{
-        |> Map.put(:pack_file, pack_file)
+            pack_file: pack_file,
-        |> Map.put(:path, Path.dirname(pack_file))
+            path: Path.dirname(pack_file),
-        |> Map.put(:name, name)
+            name: name
+          }
+        )
 
       files_count =
         pack.files
@@ -301,8 +305,6 @@ defmodule Pleroma.Emoji.Pack do
         |> length()
 
       {:ok, Map.put(pack, :files_count, files_count)}
-    else
-      {:error, :not_found}
     end
   end
 
@@ -415,7 +417,7 @@ defmodule Pleroma.Emoji.Pack do
     ttl_per_file = Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file])
     overall_ttl = :timer.seconds(ttl_per_file * Enum.count(files))
 
-    Cachex.put!(
+    @cachex.put(
       :emoji_packs_cache,
       pack.name,
       # if pack.json MD5 changes, the cache is not valid anymore
@@ -434,10 +436,17 @@ defmodule Pleroma.Emoji.Pack do
     end
   end
 
-  defp from_json(json) do
+  defp from_json(json, attrs) do
     map = Jason.decode!(json)
 
-    struct(__MODULE__, %{files: map["files"], pack: map["pack"]})
+    pack_attrs =
+      attrs
+      |> Map.merge(%{
+        files: map["files"],
+        pack: map["pack"]
+      })
+
+    struct(__MODULE__, pack_attrs)
   end
 
   defp validate_shareable_packs_available(uri) do
@@ -491,10 +500,10 @@ defmodule Pleroma.Emoji.Pack do
   end
 
   defp create_subdirs(file_path) do
-    if String.contains?(file_path, "/") do
+    with true <- String.contains?(file_path, "/"),
-      file_path
+         path <- Path.dirname(file_path),
-      |> Path.dirname()
+         false <- File.exists?(path) do
-      |> File.mkdir_p!()
+      File.mkdir_p!(path)
     end
   end
 
@@ -518,10 +527,15 @@ defmodule Pleroma.Emoji.Pack do
 
   defp get_filename(pack, shortcode) do
     with %{^shortcode => filename} when is_binary(filename) <- pack.files,
-         true <- pack.path |> Path.join(filename) |> File.exists?() do
+         file_path <- Path.join(pack.path, filename),
+         {:ok, _} <- File.stat(file_path) do
       {:ok, filename}
     else
-      _ -> {:error, :doesnt_exist}
+      {:error, _} = error ->
+        error
+
+      _ ->
+        {:error, :doesnt_exist}
     end
   end
 
@@ -606,7 +620,7 @@ defmodule Pleroma.Emoji.Pack do
   defp fetch_archive(pack) do
     hash = :crypto.hash(:md5, File.read!(pack.pack_file))
 
-    case Cachex.get!(:emoji_packs_cache, pack.name) do
+    case @cachex.get!(:emoji_packs_cache, pack.name) do
       %{hash: ^hash, pack_data: archive} -> archive
       _ -> create_archive_and_cache(pack, hash)
     end

lib/pleroma/following_relationship.ex

@@ -62,23 +62,47 @@ defmodule Pleroma.FollowingRelationship do
         follow(follower, following, state)
 
       following_relationship ->
+        with {:ok, _following_relationship} <-
                following_relationship
                |> cast(%{state: state}, [:state])
                |> validate_required([:state])
-        |> Repo.update()
+               |> Repo.update() do
+          after_update(state, follower, following)
+        end
     end
   end
 
   def follow(%User{} = follower, %User{} = following, state \\ :follow_accept) do
+    with {:ok, _following_relationship} <-
            %__MODULE__{}
            |> changeset(%{follower: follower, following: following, state: state})
-    |> Repo.insert(on_conflict: :nothing)
+           |> Repo.insert(on_conflict: :nothing) do
+      after_update(state, follower, following)
+    end
   end
 
   def unfollow(%User{} = follower, %User{} = following) do
     case get(follower, following) do
-      %__MODULE__{} = following_relationship -> Repo.delete(following_relationship)
+      %__MODULE__{} = following_relationship ->
-      _ -> {:ok, nil}
+        with {:ok, _following_relationship} <- Repo.delete(following_relationship) do
+          after_update(:unfollow, follower, following)
+        end
+
+      _ ->
+        {:ok, nil}
+    end
+  end
+
+  defp after_update(state, %User{} = follower, %User{} = following) do
+    with {:ok, following} <- User.update_follower_count(following),
+         {:ok, follower} <- User.update_following_count(follower) do
+      Pleroma.Web.Streamer.stream("follow_relationship", %{
+        state: state,
+        following: following,
+        follower: follower
+      })
+
+      {:ok, follower, following}
     end
   end
 

lib/pleroma/frontend.ex

@@ -0,0 +1,110 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Frontend do
+  alias Pleroma.Config
+
+  require Logger
+
+  def install(name, opts \\ []) do
+    frontend_info = %{
+      "ref" => opts[:ref],
+      "build_url" => opts[:build_url],
+      "build_dir" => opts[:build_dir]
+    }
+
+    frontend_info =
+      [:frontends, :available, name]
+      |> Config.get(%{})
+      |> Map.merge(frontend_info, fn _key, config, cmd ->
+        # This only overrides things that are actually set
+        cmd || config
+      end)
+
+    ref = frontend_info["ref"]
+
+    unless ref do
+      raise "No ref given or configured"
+    end
+
+    dest = Path.join([dir(), name, ref])
+
+    label = "#{name} (#{ref})"
+    tmp_dir = Path.join(dir(), "tmp")
+
+    with {_, :ok} <-
+           {:download_or_unzip, download_or_unzip(frontend_info, tmp_dir, opts[:file])},
+         Logger.info("Installing #{label} to #{dest}"),
+         :ok <- install_frontend(frontend_info, tmp_dir, dest) do
+      File.rm_rf!(tmp_dir)
+      Logger.info("Frontend #{label} installed to #{dest}")
+    else
+      {:download_or_unzip, _} ->
+        Logger.info("Could not download or unzip the frontend")
+        {:error, "Could not download or unzip the frontend"}
+
+      _e ->
+        Logger.info("Could not install the frontend")
+        {:error, "Could not install the frontend"}
+    end
+  end
+
+  def dir(opts \\ []) do
+    if is_nil(opts[:static_dir]) do
+      Pleroma.Config.get!([:instance, :static_dir])
+    else
+      opts[:static_dir]
+    end
+    |> Path.join("frontends")
+  end
+
+  defp download_or_unzip(frontend_info, temp_dir, nil),
+    do: download_build(frontend_info, temp_dir)
+
+  defp download_or_unzip(_frontend_info, temp_dir, file) do
+    with {:ok, zip} <- File.read(Path.expand(file)) do
+      unzip(zip, temp_dir)
+    end
+  end
+
+  def unzip(zip, dest) do
+    with {:ok, unzipped} <- :zip.unzip(zip, [:memory]) do
+      File.rm_rf!(dest)
+      File.mkdir_p!(dest)
+
+      Enum.each(unzipped, fn {filename, data} ->
+        path = filename
+
+        new_file_path = Path.join(dest, path)
+
+        new_file_path
+        |> Path.dirname()
+        |> File.mkdir_p!()
+
+        File.write!(new_file_path, data)
+      end)
+    end
+  end
+
+  defp download_build(frontend_info, dest) do
+    Logger.info("Downloading pre-built bundle for #{frontend_info["name"]}")
+    url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
+
+    with {:ok, %{status: 200, body: zip_body}} <-
+           Pleroma.HTTP.get(url, [], pool: :media, recv_timeout: 120_000) do
+      unzip(zip_body, dest)
+    else
+      {:error, e} -> {:error, e}
+      e -> {:error, e}
+    end
+  end
+
+  defp install_frontend(frontend_info, source, dest) do
+    from = frontend_info["build_dir"] || "dist"
+    File.rm_rf!(dest)
+    File.mkdir_p!(dest)
+    File.cp_r!(Path.join([source, from]), dest)
+    :ok
+  end
+end

lib/pleroma/helpers/auth_helper.ex

@@ -0,0 +1,46 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Helpers.AuthHelper do
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
+  alias Plug.Conn
+
+  import Plug.Conn
+
+  @oauth_token_session_key :oauth_token
+
+  @doc """
+  Skips OAuth permissions (scopes) checks, assigns nil `:token`.
+  Intended to be used with explicit authentication and only when OAuth token cannot be determined.
+  """
+  def skip_oauth(conn) do
+    conn
+    |> assign(:token, nil)
+    |> OAuthScopesPlug.skip_plug()
+  end
+
+  @doc "Drops authentication info from connection"
+  def drop_auth_info(conn) do
+    # To simplify debugging, setting a private variable on `conn` if auth info is dropped
+    conn
+    |> assign(:user, nil)
+    |> assign(:token, nil)
+    |> put_private(:authentication_ignored, true)
+  end
+
+  @doc "Gets OAuth token string from session"
+  def get_session_token(%Conn{} = conn) do
+    get_session(conn, @oauth_token_session_key)
+  end
+
+  @doc "Updates OAuth token string in session"
+  def put_session_token(%Conn{} = conn, token) when is_binary(token) do
+    put_session(conn, @oauth_token_session_key, token)
+  end
+
+  @doc "Deletes OAuth token string from session"
+  def delete_session_token(%Conn{} = conn) do
+    delete_session(conn, @oauth_token_session_key)
+  end
+end

lib/pleroma/html.ex

@@ -6,6 +6,8 @@ defmodule Pleroma.HTML do
   # Scrubbers are compiled on boot so they can be configured in OTP releases
   #  @on_load :compile_scrubbers
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
   def compile_scrubbers do
     dir = Path.join(:code.priv_dir(:pleroma), "scrubbers")
 
@@ -56,7 +58,7 @@ defmodule Pleroma.HTML do
       ) do
     key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}"
 
-    Cachex.fetch!(:scrubber_cache, key, fn _key ->
+    @cachex.fetch!(:scrubber_cache, key, fn _key ->
       object = Pleroma.Object.normalize(activity)
       ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback)
     end)
@@ -105,7 +107,7 @@ defmodule Pleroma.HTML do
     unless object.data["fake"] do
       key = "URL|#{object.id}"
 
-      Cachex.fetch!(:scrubber_cache, key, fn _key ->
+      @cachex.fetch!(:scrubber_cache, key, fn _key ->
         {:commit, {:ok, extract_first_external_url(content)}}
       end)
     else

lib/pleroma/instances/instance.ex

@@ -77,7 +77,7 @@ defmodule Pleroma.Instances.Instance do
     )
   end
 
-  def reachable?(_), do: true
+  def reachable?(url_or_host) when is_binary(url_or_host), do: true
 
   def set_reachable(url_or_host) when is_binary(url_or_host) do
     with host <- host(url_or_host),
@@ -166,7 +166,8 @@ defmodule Pleroma.Instances.Instance do
 
   defp scrape_favicon(%URI{} = instance_uri) do
     try do
-      with {:ok, %Tesla.Env{body: html}} <-
+      with {_, true} <- {:reachable, reachable?(instance_uri.host)},
+           {:ok, %Tesla.Env{body: html}} <-
              Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], pool: :media),
            {_, [favicon_rel | _]} when is_binary(favicon_rel) <-
              {:parse,
@@ -175,7 +176,15 @@ defmodule Pleroma.Instances.Instance do
              {:merge, URI.merge(instance_uri, favicon_rel) |> to_string()} do
         favicon
       else
-        _ -> nil
+        {:reachable, false} ->
+          Logger.debug(
+            "Instance.scrape_favicon(\"#{to_string(instance_uri)}\") ignored unreachable host"
+          )
+
+          nil
+
+        _ ->
+          nil
       end
     rescue
       e ->

lib/pleroma/moderation_log.ex

@@ -12,6 +12,26 @@ defmodule Pleroma.ModerationLog do
 
   import Ecto.Query
 
+  @type t :: %__MODULE__{}
+  @type log_subject :: Activity.t() | User.t() | list(User.t())
+  @type log_params :: %{
+          required(:actor) => User.t(),
+          required(:action) => String.t(),
+          optional(:subject) => log_subject(),
+          optional(:subject_actor) => User.t(),
+          optional(:subject_id) => String.t(),
+          optional(:subjects) => list(User.t()),
+          optional(:permission) => String.t(),
+          optional(:text) => String.t(),
+          optional(:sensitive) => String.t(),
+          optional(:visibility) => String.t(),
+          optional(:followed) => User.t(),
+          optional(:follower) => User.t(),
+          optional(:nicknames) => list(String.t()),
+          optional(:tags) => list(String.t()),
+          optional(:target) => String.t()
+        }
+
   schema "moderation_log" do
     field(:data, :map)
 
@@ -90,203 +110,105 @@ defmodule Pleroma.ModerationLog do
     parsed_datetime
   end
 
-  @spec insert_log(%{actor: User, subject: [User], action: String.t(), permission: String.t()}) ::
+  defp prepare_log_data(%{actor: actor, action: action} = attrs) do
-          {:ok, ModerationLog} | {:error, any}
+    %{
-  def insert_log(%{
-        actor: %User{} = actor,
-        subject: subjects,
-        action: action,
-        permission: permission
-      }) do
-    %ModerationLog{
-      data: %{
       "actor" => user_to_map(actor),
-        "subject" => user_to_map(subjects),
       "action" => action,
-        "permission" => permission,
       "message" => ""
     }
-    }
+    |> Pleroma.Maps.put_if_present("subject_actor", user_to_map(attrs[:subject_actor]))
-    |> insert_log_entry_with_message()
   end
 
-  @spec insert_log(%{actor: User, subject: User, action: String.t()}) ::
+  defp prepare_log_data(attrs), do: attrs
-          {:ok, ModerationLog} | {:error, any}
+
-  def insert_log(%{
+  @spec insert_log(log_params()) :: {:ok, ModerationLog} | {:error, any}
-        actor: %User{} = actor,
+  def insert_log(%{actor: %User{}, subject: subjects, permission: permission} = attrs) do
-        action: "report_update",
+    data =
-        subject: %Activity{data: %{"type" => "Flag"}} = subject
+      attrs
-      }) do
+      |> prepare_log_data
-    %ModerationLog{
+      |> Map.merge(%{"subject" => user_to_map(subjects), "permission" => permission})
-      data: %{
+
-        "actor" => user_to_map(actor),
+    insert_log_entry_with_message(%ModerationLog{data: data})
-        "action" => "report_update",
-        "subject" => report_to_map(subject),
-        "message" => ""
-      }
-    }
-    |> insert_log_entry_with_message()
   end
 
-  @spec insert_log(%{actor: User, subject: Activity, action: String.t(), text: String.t()}) ::
+  def insert_log(%{actor: %User{}, action: action, subject: %Activity{} = subject} = attrs)
-          {:ok, ModerationLog} | {:error, any}
+      when action in ["report_note_delete", "report_update", "report_note"] do
-  def insert_log(%{
+    data =
-        actor: %User{} = actor,
+      attrs
-        action: "report_note",
+      |> prepare_log_data
-        subject: %Activity{} = subject,
+      |> Pleroma.Maps.put_if_present("text", attrs[:text])
-        text: text
+      |> Map.merge(%{"subject" => report_to_map(subject)})
-      }) do
+
-    %ModerationLog{
+    insert_log_entry_with_message(%ModerationLog{data: data})
-      data: %{
-        "actor" => user_to_map(actor),
-        "action" => "report_note",
-        "subject" => report_to_map(subject),
-        "text" => text
-      }
-    }
-    |> insert_log_entry_with_message()
   end
 
-  @spec insert_log(%{actor: User, subject: Activity, action: String.t(), text: String.t()}) ::
+  def insert_log(
-          {:ok, ModerationLog} | {:error, any}
+        %{
-  def insert_log(%{
+          actor: %User{},
-        actor: %User{} = actor,
+          action: action,
-        action: "report_note_delete",
-        subject: %Activity{} = subject,
-        text: text
-      }) do
-    %ModerationLog{
-      data: %{
-        "actor" => user_to_map(actor),
-        "action" => "report_note_delete",
-        "subject" => report_to_map(subject),
-        "text" => text
-      }
-    }
-    |> insert_log_entry_with_message()
-  end
-
-  @spec insert_log(%{
-          actor: User,
-          subject: Activity,
-          action: String.t(),
-          sensitive: String.t(),
-          visibility: String.t()
-        }) :: {:ok, ModerationLog} | {:error, any}
-  def insert_log(%{
-        actor: %User{} = actor,
-        action: "status_update",
           subject: %Activity{} = subject,
           sensitive: sensitive,
           visibility: visibility
-      }) do
+        } = attrs
-    %ModerationLog{
+      )
-      data: %{
+      when action == "status_update" do
-        "actor" => user_to_map(actor),
+    data =
-        "action" => "status_update",
+      attrs
+      |> prepare_log_data
+      |> Map.merge(%{
         "subject" => status_to_map(subject),
         "sensitive" => sensitive,
-        "visibility" => visibility,
+        "visibility" => visibility
-        "message" => ""
+      })
-      }
+
-    }
+    insert_log_entry_with_message(%ModerationLog{data: data})
-    |> insert_log_entry_with_message()
   end
 
-  @spec insert_log(%{actor: User, action: String.t(), subject_id: String.t()}) ::
+  def insert_log(%{actor: %User{}, action: action, subject_id: subject_id} = attrs)
-          {:ok, ModerationLog} | {:error, any}
+      when action == "status_delete" do
-  def insert_log(%{
+    data =
-        actor: %User{} = actor,
+      attrs
-        action: "status_delete",
+      |> prepare_log_data
-        subject_id: subject_id
+      |> Map.merge(%{"subject_id" => subject_id})
-      }) do
+
-    %ModerationLog{
+    insert_log_entry_with_message(%ModerationLog{data: data})
-      data: %{
-        "actor" => user_to_map(actor),
-        "action" => "status_delete",
-        "subject_id" => subject_id,
-        "message" => ""
-      }
-    }
-    |> insert_log_entry_with_message()
   end
 
-  @spec insert_log(%{actor: User, subject: User, action: String.t()}) ::
+  def insert_log(%{actor: %User{}, subject: subject, action: _action} = attrs) do
-          {:ok, ModerationLog} | {:error, any}
+    data =
-  def insert_log(%{actor: %User{} = actor, subject: subject, action: action}) do
+      attrs
-    %ModerationLog{
+      |> prepare_log_data
-      data: %{
+      |> Map.merge(%{"subject" => user_to_map(subject)})
-        "actor" => user_to_map(actor),
+
-        "action" => action,
+    insert_log_entry_with_message(%ModerationLog{data: data})
-        "subject" => user_to_map(subject),
-        "message" => ""
-      }
-    }
-    |> insert_log_entry_with_message()
   end
 
-  @spec insert_log(%{actor: User, subjects: [User], action: String.t()}) ::
+  def insert_log(%{actor: %User{}, subjects: subjects, action: _action} = attrs) do
-          {:ok, ModerationLog} | {:error, any}
+    data =
-  def insert_log(%{actor: %User{} = actor, subjects: subjects, action: action}) do
+      attrs
-    subjects = Enum.map(subjects, &user_to_map/1)
+      |> prepare_log_data
+      |> Map.merge(%{"subjects" => user_to_map(subjects)})
 
-    %ModerationLog{
+    insert_log_entry_with_message(%ModerationLog{data: data})
-      data: %{
-        "actor" => user_to_map(actor),
-        "action" => action,
-        "subjects" => subjects,
-        "message" => ""
-      }
-    }
-    |> insert_log_entry_with_message()
   end
 
-  @spec insert_log(%{actor: User, action: String.t(), followed: User, follower: User}) ::
+  def insert_log(
-          {:ok, ModerationLog} | {:error, any}
+        %{
-  def insert_log(%{
+          actor: %User{},
-        actor: %User{} = actor,
           followed: %User{} = followed,
           follower: %User{} = follower,
-        action: "follow"
+          action: action
-      }) do
+        } = attrs
-    %ModerationLog{
+      )
-      data: %{
+      when action in ["unfollow", "follow"] do
-        "actor" => user_to_map(actor),
+    data =
-        "action" => "follow",
+      attrs
-        "followed" => user_to_map(followed),
+      |> prepare_log_data
-        "follower" => user_to_map(follower),
+      |> Map.merge(%{"followed" => user_to_map(followed), "follower" => user_to_map(follower)})
-        "message" => ""
+
-      }
+    insert_log_entry_with_message(%ModerationLog{data: data})
-    }
-    |> insert_log_entry_with_message()
   end
 
-  @spec insert_log(%{actor: User, action: String.t(), followed: User, follower: User}) ::
-          {:ok, ModerationLog} | {:error, any}
-  def insert_log(%{
-        actor: %User{} = actor,
-        followed: %User{} = followed,
-        follower: %User{} = follower,
-        action: "unfollow"
-      }) do
-    %ModerationLog{
-      data: %{
-        "actor" => user_to_map(actor),
-        "action" => "unfollow",
-        "followed" => user_to_map(followed),
-        "follower" => user_to_map(follower),
-        "message" => ""
-      }
-    }
-    |> insert_log_entry_with_message()
-  end
-
-  @spec insert_log(%{
-          actor: User,
-          action: String.t(),
-          nicknames: [String.t()],
-          tags: [String.t()]
-        }) :: {:ok, ModerationLog} | {:error, any}
   def insert_log(%{
         actor: %User{} = actor,
         nicknames: nicknames,
@@ -305,27 +227,16 @@ defmodule Pleroma.ModerationLog do
     |> insert_log_entry_with_message()
   end
 
-  @spec insert_log(%{actor: User, action: String.t(), target: String.t()}) ::
+  def insert_log(%{actor: %User{}, action: action, target: target} = attrs)
-          {:ok, ModerationLog} | {:error, any}
-  def insert_log(%{
-        actor: %User{} = actor,
-        action: action,
-        target: target
-      })
       when action in ["relay_follow", "relay_unfollow"] do
-    %ModerationLog{
+    data =
-      data: %{
+      attrs
-        "actor" => user_to_map(actor),
+      |> prepare_log_data
-        "action" => action,
+      |> Map.merge(%{"target" => target})
-        "target" => target,
+
-        "message" => ""
+    insert_log_entry_with_message(%ModerationLog{data: data})
-      }
-    }
-    |> insert_log_entry_with_message()
   end
 
-  @spec insert_log(%{actor: User, action: String.t(), subject_id: String.t()}) ::
-          {:ok, ModerationLog} | {:error, any}
   def insert_log(%{actor: %User{} = actor, action: "chat_message_delete", subject_id: subject_id}) do
     %ModerationLog{
       data: %{
@@ -345,32 +256,27 @@ defmodule Pleroma.ModerationLog do
   end
 
   defp user_to_map(users) when is_list(users) do
-    users |> Enum.map(&user_to_map/1)
+    Enum.map(users, &user_to_map/1)
   end
 
   defp user_to_map(%User{} = user) do
     user
-    |> Map.from_struct()
     |> Map.take([:id, :nickname])
     |> Map.new(fn {k, v} -> {Atom.to_string(k), v} end)
     |> Map.put("type", "user")
   end
 
+  defp user_to_map(_), do: nil
+
   defp report_to_map(%Activity{} = report) do
-    %{
+    %{"type" => "report", "id" => report.id, "state" => report.data["state"]}
-      "type" => "report",
-      "id" => report.id,
-      "state" => report.data["state"]
-    }
   end
 
   defp status_to_map(%Activity{} = status) do
-    %{
+    %{"type" => "status", "id" => status.id}
-      "type" => "status",
-      "id" => status.id
-    }
   end
 
+  @spec get_log_entry_message(ModerationLog.t()) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -382,7 +288,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} made @#{follower_nickname} #{action} @#{followed_nickname}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -393,7 +298,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} deleted users: #{users_to_nicknames_string(subjects)}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -404,7 +308,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} created users: #{users_to_nicknames_string(subjects)}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -415,7 +318,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} activated users: #{users_to_nicknames_string(users)}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -426,7 +328,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} deactivated users: #{users_to_nicknames_string(users)}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -437,7 +338,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} approved users: #{users_to_nicknames_string(users)}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -451,7 +351,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} added tags: #{tags_string} to users: #{nicknames_to_string(nicknames)}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -465,7 +364,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} removed tags: #{tags_string} from users: #{nicknames_to_string(nicknames)}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -477,7 +375,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} made #{users_to_nicknames_string(users)} #{permission}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -489,7 +386,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} revoked #{permission} role from #{users_to_nicknames_string(users)}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -500,7 +396,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} followed relay: #{target}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -511,42 +406,48 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} unfollowed relay: #{target}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
+  def get_log_entry_message(
-  def get_log_entry_message(%ModerationLog{
+        %ModerationLog{
           data: %{
             "actor" => %{"nickname" => actor_nickname},
             "action" => "report_update",
             "subject" => %{"id" => subject_id, "state" => state, "type" => "report"}
           }
-      }) do
+        } = log
-    "@#{actor_nickname} updated report ##{subject_id} with '#{state}' state"
+      ) do
+    "@#{actor_nickname} updated report ##{subject_id}" <>
+      subject_actor_nickname(log, " (on user ", ")") <>
+      " with '#{state}' state"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
+  def get_log_entry_message(
-  def get_log_entry_message(%ModerationLog{
+        %ModerationLog{
           data: %{
             "actor" => %{"nickname" => actor_nickname},
             "action" => "report_note",
             "subject" => %{"id" => subject_id, "type" => "report"},
             "text" => text
           }
-      }) do
+        } = log
-    "@#{actor_nickname} added note '#{text}' to report ##{subject_id}"
+      ) do
+    "@#{actor_nickname} added note '#{text}' to report ##{subject_id}" <>
+      subject_actor_nickname(log, " on user ")
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
+  def get_log_entry_message(
-  def get_log_entry_message(%ModerationLog{
+        %ModerationLog{
           data: %{
             "actor" => %{"nickname" => actor_nickname},
             "action" => "report_note_delete",
             "subject" => %{"id" => subject_id, "type" => "report"},
             "text" => text
           }
-      }) do
+        } = log
-    "@#{actor_nickname} deleted note '#{text}' from report ##{subject_id}"
+      ) do
+    "@#{actor_nickname} deleted note '#{text}' from report ##{subject_id}" <>
+      subject_actor_nickname(log, " on user ")
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -559,7 +460,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} updated status ##{subject_id}, set visibility: '#{visibility}'"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -572,7 +472,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} updated status ##{subject_id}, set sensitive: '#{sensitive}'"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -587,7 +486,6 @@ defmodule Pleroma.ModerationLog do
     }'"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -598,7 +496,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} deleted status ##{subject_id}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -609,7 +506,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} forced password reset for users: #{users_to_nicknames_string(subjects)}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -620,7 +516,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} confirmed email for users: #{users_to_nicknames_string(subjects)}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -633,7 +528,6 @@ defmodule Pleroma.ModerationLog do
     }"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -644,7 +538,6 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} updated users: #{users_to_nicknames_string(subjects)}"
   end
 
-  @spec get_log_entry_message(ModerationLog) :: String.t()
   def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
@@ -676,4 +569,16 @@ defmodule Pleroma.ModerationLog do
     |> Enum.map(&"@#{&1["nickname"]}")
     |> Enum.join(", ")
   end
+
+  defp subject_actor_nickname(%ModerationLog{data: data}, prefix_msg, postfix_msg \\ "") do
+    case data do
+      %{"subject_actor" => %{"nickname" => subject_actor}} ->
+        [prefix_msg, "@#{subject_actor}", postfix_msg]
+        |> Enum.reject(&(&1 == ""))
+        |> Enum.join()
+
+      _ ->
+        ""
+    end
+  end
 end

lib/pleroma/object.ex

@@ -23,6 +23,8 @@ defmodule Pleroma.Object do
 
   @derive {Jason.Encoder, only: [:data]}
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
   schema "objects" do
     field(:data, :map)
 
@@ -156,9 +158,9 @@ defmodule Pleroma.Object do
   def get_cached_by_ap_id(ap_id) do
     key = "object:#{ap_id}"
 
-    with {:ok, nil} <- Cachex.get(:object_cache, key),
+    with {:ok, nil} <- @cachex.get(:object_cache, key),
          object when not is_nil(object) <- get_by_ap_id(ap_id),
-         {:ok, true} <- Cachex.put(:object_cache, key, object) do
+         {:ok, true} <- @cachex.put(:object_cache, key, object) do
       object
     else
       {:ok, object} -> object
@@ -216,13 +218,13 @@ defmodule Pleroma.Object do
   end
 
   def invalid_object_cache(%Object{data: %{"id" => id}}) do
-    with {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
+    with {:ok, true} <- @cachex.del(:object_cache, "object:#{id}") do
-      Cachex.del(:web_resp_cache, URI.parse(id).path)
+      @cachex.del(:web_resp_cache, URI.parse(id).path)
     end
   end
 
   def set_cache(%Object{data: %{"id" => ap_id}} = object) do
-    Cachex.put(:object_cache, "object:#{ap_id}", object)
+    @cachex.put(:object_cache, "object:#{ap_id}", object)
     {:ok, object}
   end
 

lib/pleroma/object/fetcher.ex

@@ -12,7 +12,6 @@ defmodule Pleroma.Object.Fetcher do
   alias Pleroma.Web.ActivityPub.ObjectValidator
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.Federator
-  alias Pleroma.Web.FedSockets
 
   require Logger
   require Pleroma.Constants
@@ -183,16 +182,16 @@ defmodule Pleroma.Object.Fetcher do
     end
   end
 
-  def fetch_and_contain_remote_object_from_id(prm, opts \\ [])
+  def fetch_and_contain_remote_object_from_id(id)
 
-  def fetch_and_contain_remote_object_from_id(%{"id" => id}, opts),
+  def fetch_and_contain_remote_object_from_id(%{"id" => id}),
-    do: fetch_and_contain_remote_object_from_id(id, opts)
+    do: fetch_and_contain_remote_object_from_id(id)
 
-  def fetch_and_contain_remote_object_from_id(id, opts) when is_binary(id) do
+  def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
     Logger.debug("Fetching object #{id} via AP")
 
     with {:scheme, true} <- {:scheme, String.starts_with?(id, "http")},
-         {:ok, body} <- get_object(id, opts),
+         {:ok, body} <- get_object(id),
          {:ok, data} <- safe_json_decode(body),
          :ok <- Containment.contain_origin_from_id(id, data) do
       {:ok, data}
@@ -208,22 +207,10 @@ defmodule Pleroma.Object.Fetcher do
     end
   end
 
-  def fetch_and_contain_remote_object_from_id(_id, _opts),
+  def fetch_and_contain_remote_object_from_id(_id),
     do: {:error, "id must be a string"}
 
-  defp get_object(id, opts) do
+  defp get_object(id) do
-    with false <- Keyword.get(opts, :force_http, false),
-         {:ok, fedsocket} <- FedSockets.get_or_create_fed_socket(id) do
-      Logger.debug("fetching via fedsocket - #{inspect(id)}")
-      FedSockets.fetch(fedsocket, id)
-    else
-      _other ->
-        Logger.debug("fetching via http - #{inspect(id)}")
-        get_object_http(id)
-    end
-  end
-
-  defp get_object_http(id) do
     date = Pleroma.Signature.signed_date()
 
     headers =

lib/pleroma/password_reset_token.ex

@@ -40,6 +40,7 @@ defmodule Pleroma.PasswordResetToken do
   @spec reset_password(binary(), map()) :: {:ok, User.t()} | {:error, binary()}
   def reset_password(token, data) do
     with %{used: false} = token <- Repo.get_by(PasswordResetToken, %{token: token}),
+         false <- expired?(token),
          %User{} = user <- User.get_cached_by_id(token.user_id),
          {:ok, _user} <- User.reset_password(user, data),
          {:ok, token} <- Repo.update(used_changeset(token)) do
@@ -48,4 +49,14 @@ defmodule Pleroma.PasswordResetToken do
       _e -> {:error, token}
     end
   end
+
+  def expired?(%__MODULE__{inserted_at: inserted_at}) do
+    validity = Pleroma.Config.get([:instance, :password_reset_token_validity], 0)
+
+    now = NaiveDateTime.utc_now()
+
+    difference = NaiveDateTime.diff(now, inserted_at)
+
+    difference > validity
+  end
 end

lib/pleroma/reverse_proxy.ex

@@ -17,6 +17,8 @@ defmodule Pleroma.ReverseProxy do
   @failed_request_ttl :timer.seconds(60)
   @methods ~w(GET HEAD)
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
   def max_read_duration_default, do: @max_read_duration
   def default_cache_control_header, do: @default_cache_control_header
 
@@ -107,7 +109,7 @@ defmodule Pleroma.ReverseProxy do
         opts
       end
 
-    with {:ok, nil} <- Cachex.get(:failed_proxy_url_cache, url),
+    with {:ok, nil} <- @cachex.get(:failed_proxy_url_cache, url),
          {:ok, code, headers, client} <- request(method, url, req_headers, client_opts),
          :ok <-
            header_length_constraint(
@@ -427,6 +429,6 @@ defmodule Pleroma.ReverseProxy do
         nil
       end
 
-    Cachex.put(:failed_proxy_url_cache, url, true, ttl: ttl)
+    @cachex.put(:failed_proxy_url_cache, url, true, ttl: ttl)
   end
 end

lib/pleroma/signature.ex

@@ -39,7 +39,7 @@ defmodule Pleroma.Signature do
   def fetch_public_key(conn) do
     with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
          {:ok, actor_id} <- key_id_to_actor_id(kid),
-         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id, force_http: true) do
+         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
       {:ok, public_key}
     else
       e ->
@@ -50,8 +50,8 @@ defmodule Pleroma.Signature do
   def refetch_public_key(conn) do
     with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
          {:ok, actor_id} <- key_id_to_actor_id(kid),
-         {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id, force_http: true),
+         {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id),
-         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id, force_http: true) do
+         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
       {:ok, public_key}
     else
       e ->

lib/pleroma/user.ex

@@ -81,6 +81,8 @@ defmodule Pleroma.User do
     ]
   ]
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
   schema "users" do
     field(:bio, :string, default: "")
     field(:raw_bio, :string)
@@ -245,6 +247,18 @@ defmodule Pleroma.User do
     end
   end
 
+  def cached_blocked_users_ap_ids(user) do
+    @cachex.fetch!(:user_cache, "blocked_users_ap_ids:#{user.ap_id}", fn _ ->
+      blocked_users_ap_ids(user)
+    end)
+  end
+
+  def cached_muted_users_ap_ids(user) do
+    @cachex.fetch!(:user_cache, "muted_users_ap_ids:#{user.ap_id}", fn _ ->
+      muted_users_ap_ids(user)
+    end)
+  end
+
   defdelegate following_count(user), to: FollowingRelationship
   defdelegate following(user), to: FollowingRelationship
   defdelegate following?(follower, followed), to: FollowingRelationship
@@ -461,6 +475,18 @@ defmodule Pleroma.User do
     |> validate_length(:bio, max: bio_limit)
     |> validate_length(:name, max: name_limit)
     |> validate_fields(true)
+    |> validate_non_local()
+  end
+
+  defp validate_non_local(cng) do
+    local? = get_field(cng, :local)
+
+    if local? do
+      cng
+      |> add_error(:local, "User is local, can't update with this changeset.")
+    else
+      cng
+    end
   end
 
   def update_changeset(struct, params \\ %{}) do
@@ -782,18 +808,50 @@ defmodule Pleroma.User do
     end
   end
 
-  def post_register_action(%User{} = user) do
+  def post_register_action(%User{confirmation_pending: true} = user) do
+    with {:ok, _} <- try_send_confirmation_email(user) do
+      {:ok, user}
+    end
+  end
+
+  def post_register_action(%User{approval_pending: true} = user) do
+    with {:ok, _} <- send_user_approval_email(user),
+         {:ok, _} <- send_admin_approval_emails(user) do
+      {:ok, user}
+    end
+  end
+
+  def post_register_action(%User{approval_pending: false, confirmation_pending: false} = user) do
     with {:ok, user} <- autofollow_users(user),
          {:ok, _} <- autofollowing_users(user),
          {:ok, user} <- set_cache(user),
          {:ok, _} <- send_welcome_email(user),
          {:ok, _} <- send_welcome_message(user),
-         {:ok, _} <- send_welcome_chat_message(user),
+         {:ok, _} <- send_welcome_chat_message(user) do
-         {:ok, _} <- try_send_confirmation_email(user) do
       {:ok, user}
     end
   end
 
+  defp send_user_approval_email(user) do
+    user
+    |> Pleroma.Emails.UserEmail.approval_pending_email()
+    |> Pleroma.Emails.Mailer.deliver_async()
+
+    {:ok, :enqueued}
+  end
+
+  defp send_admin_approval_emails(user) do
+    all_superusers()
+    |> Enum.filter(fn user -> not is_nil(user.email) end)
+    |> Enum.each(fn superuser ->
+      superuser
+      |> Pleroma.Emails.AdminEmail.new_unapproved_registration(user)
+      |> Pleroma.Emails.Mailer.deliver_async()
+    end)
+
+    {:ok, :enqueued}
+  end
+
   def send_welcome_message(user) do
     if User.WelcomeMessage.enabled?() do
       User.WelcomeMessage.post_message(user)
@@ -870,7 +928,7 @@ defmodule Pleroma.User do
     if not ap_enabled?(followed) do
       follow(follower, followed)
     else
-      {:ok, follower}
+      {:ok, follower, followed}
     end
   end
 
@@ -896,11 +954,6 @@ defmodule Pleroma.User do
 
       true ->
         FollowingRelationship.follow(follower, followed, state)
-
-        {:ok, _} = update_follower_count(followed)
-
-        follower
-        |> update_following_count()
     end
   end
 
@@ -924,11 +977,6 @@ defmodule Pleroma.User do
     case get_follow_state(follower, followed) do
       state when state in [:follow_pending, :follow_accept] ->
         FollowingRelationship.unfollow(follower, followed)
-        {:ok, followed} = update_follower_count(followed)
-
-        {:ok, follower} = update_following_count(follower)
-
-        {:ok, follower, followed}
 
       nil ->
         {:error, "Not subscribed!"}
@@ -1002,9 +1050,9 @@ defmodule Pleroma.User do
   def set_cache({:error, err}), do: {:error, err}
 
   def set_cache(%User{} = user) do
-    Cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
+    @cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
-    Cachex.put(:user_cache, "nickname:#{user.nickname}", user)
+    @cachex.put(:user_cache, "nickname:#{user.nickname}", user)
-    Cachex.put(:user_cache, "friends_ap_ids:#{user.nickname}", get_user_friends_ap_ids(user))
+    @cachex.put(:user_cache, "friends_ap_ids:#{user.nickname}", get_user_friends_ap_ids(user))
     {:ok, user}
   end
 
@@ -1027,24 +1075,26 @@ defmodule Pleroma.User do
 
   @spec get_cached_user_friends_ap_ids(User.t()) :: [String.t()]
   def get_cached_user_friends_ap_ids(user) do
-    Cachex.fetch!(:user_cache, "friends_ap_ids:#{user.ap_id}", fn _ ->
+    @cachex.fetch!(:user_cache, "friends_ap_ids:#{user.ap_id}", fn _ ->
       get_user_friends_ap_ids(user)
     end)
   end
 
   def invalidate_cache(user) do
-    Cachex.del(:user_cache, "ap_id:#{user.ap_id}")
+    @cachex.del(:user_cache, "ap_id:#{user.ap_id}")
-    Cachex.del(:user_cache, "nickname:#{user.nickname}")
+    @cachex.del(:user_cache, "nickname:#{user.nickname}")
-    Cachex.del(:user_cache, "friends_ap_ids:#{user.ap_id}")
+    @cachex.del(:user_cache, "friends_ap_ids:#{user.ap_id}")
+    @cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}")
+    @cachex.del(:user_cache, "muted_users_ap_ids:#{user.ap_id}")
   end
 
   @spec get_cached_by_ap_id(String.t()) :: User.t() | nil
   def get_cached_by_ap_id(ap_id) do
     key = "ap_id:#{ap_id}"
 
-    with {:ok, nil} <- Cachex.get(:user_cache, key),
+    with {:ok, nil} <- @cachex.get(:user_cache, key),
          user when not is_nil(user) <- get_by_ap_id(ap_id),
-         {:ok, true} <- Cachex.put(:user_cache, key, user) do
+         {:ok, true} <- @cachex.put(:user_cache, key, user) do
       user
     else
       {:ok, user} -> user
@@ -1056,11 +1106,11 @@ defmodule Pleroma.User do
     key = "id:#{id}"
 
     ap_id =
-      Cachex.fetch!(:user_cache, key, fn _ ->
+      @cachex.fetch!(:user_cache, key, fn _ ->
         user = get_by_id(id)
 
         if user do
-          Cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
+          @cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
           {:commit, user.ap_id}
         else
           {:ignore, ""}
@@ -1073,7 +1123,7 @@ defmodule Pleroma.User do
   def get_cached_by_nickname(nickname) do
     key = "nickname:#{nickname}"
 
-    Cachex.fetch!(:user_cache, key, fn ->
+    @cachex.fetch!(:user_cache, key, fn _ ->
       case get_or_fetch_by_nickname(nickname) do
         {:ok, user} -> {:commit, user}
         {:error, _error} -> {:ignore, nil}
@@ -1342,6 +1392,8 @@ defmodule Pleroma.User do
         )
       end
 
+      @cachex.del(:user_cache, "muted_users_ap_ids:#{muter.ap_id}")
+
       {:ok, Enum.filter([user_mute, user_notification_mute], & &1)}
     end
   end
@@ -1350,6 +1402,7 @@ defmodule Pleroma.User do
     with {:ok, user_mute} <- UserRelationship.delete_mute(muter, mutee),
          {:ok, user_notification_mute} <-
            UserRelationship.delete_notification_mute(muter, mutee) do
+      @cachex.del(:user_cache, "muted_users_ap_ids:#{muter.ap_id}")
       {:ok, [user_mute, user_notification_mute]}
     end
   end
@@ -1571,10 +1624,33 @@ defmodule Pleroma.User do
     end)
   end
 
-  def approve(%User{} = user) do
+  def approve(%User{approval_pending: true} = user) do
-    change(user, approval_pending: false)
+    with chg <- change(user, approval_pending: false),
-    |> update_and_set_cache()
+         {:ok, user} <- update_and_set_cache(chg) do
+      post_register_action(user)
+      {:ok, user}
     end
+  end
+
+  def approve(%User{} = user), do: {:ok, user}
+
+  def confirm(users) when is_list(users) do
+    Repo.transaction(fn ->
+      Enum.map(users, fn user ->
+        with {:ok, user} <- confirm(user), do: user
+      end)
+    end)
+  end
+
+  def confirm(%User{confirmation_pending: true} = user) do
+    with chg <- confirmation_changeset(user, need_confirmation: false),
+         {:ok, user} <- update_and_set_cache(chg) do
+      post_register_action(user)
+      {:ok, user}
+    end
+  end
+
+  def confirm(%User{} = user), do: {:ok, user}
 
   def update_notification_settings(%User{} = user, settings) do
     user
@@ -1772,12 +1848,12 @@ defmodule Pleroma.User do
 
   def html_filter_policy(_), do: Config.get([:markup, :scrub_policy])
 
-  def fetch_by_ap_id(ap_id, opts \\ []), do: ActivityPub.make_user_from_ap_id(ap_id, opts)
+  def fetch_by_ap_id(ap_id), do: ActivityPub.make_user_from_ap_id(ap_id)
 
-  def get_or_fetch_by_ap_id(ap_id, opts \\ []) do
+  def get_or_fetch_by_ap_id(ap_id) do
     cached_user = get_cached_by_ap_id(ap_id)
 
-    maybe_fetched_user = needs_update?(cached_user) && fetch_by_ap_id(ap_id, opts)
+    maybe_fetched_user = needs_update?(cached_user) && fetch_by_ap_id(ap_id)
 
     case {cached_user, maybe_fetched_user} do
       {_, {:ok, %User{} = user}} ->
@@ -1850,8 +1926,8 @@ defmodule Pleroma.User do
 
   def public_key(_), do: {:error, "key not found"}
 
-  def get_public_key_for_ap_id(ap_id, opts \\ []) do
+  def get_public_key_for_ap_id(ap_id) do
-    with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id, opts),
+    with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id),
          {:ok, public_key} <- public_key(user) do
       {:ok, public_key}
     else
@@ -2062,18 +2138,6 @@ defmodule Pleroma.User do
     updated_user
   end
 
-  @spec toggle_confirmation(User.t()) :: {:ok, User.t()} | {:error, Changeset.t()}
-  def toggle_confirmation(%User{} = user) do
-    user
-    |> confirmation_changeset(need_confirmation: !user.confirmation_pending)
-    |> update_and_set_cache()
-  end
-
-  @spec toggle_confirmation([User.t()]) :: [{:ok, User.t()} | {:error, Changeset.t()}]
-  def toggle_confirmation(users) do
-    Enum.map(users, &toggle_confirmation/1)
-  end
-
   @spec need_confirmation(User.t(), boolean()) :: {:ok, User.t()} | {:error, Changeset.t()}
   def need_confirmation(%User{} = user, bool) do
     user
@@ -2345,13 +2409,19 @@ defmodule Pleroma.User do
   @spec add_to_block(User.t(), User.t()) ::
           {:ok, UserRelationship.t()} | {:error, Ecto.Changeset.t()}
   defp add_to_block(%User{} = user, %User{} = blocked) do
-    UserRelationship.create_block(user, blocked)
+    with {:ok, relationship} <- UserRelationship.create_block(user, blocked) do
+      @cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}")
+      {:ok, relationship}
+    end
   end
 
   @spec add_to_block(User.t(), User.t()) ::
           {:ok, UserRelationship.t()} | {:ok, nil} | {:error, Ecto.Changeset.t()}
   defp remove_from_block(%User{} = user, %User{} = blocked) do
-    UserRelationship.delete_block(user, blocked)
+    with {:ok, relationship} <- UserRelationship.delete_block(user, blocked) do
+      @cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}")
+      {:ok, relationship}
+    end
   end
 
   def set_invisible(user, invisible) do
@@ -2383,4 +2453,8 @@ defmodule Pleroma.User do
     |> Map.put(:bio, HTML.filter_tags(user.bio, filter))
     |> Map.put(:fields, fields)
   end
+
+  def get_host(%User{ap_id: ap_id} = _user) do
+    URI.parse(ap_id).host
+  end
 end

lib/pleroma/user/import.ex

@@ -45,7 +45,7 @@ defmodule Pleroma.User.Import do
       identifiers,
       fn identifier ->
         with {:ok, %User{} = followed} <- User.get_or_fetch(identifier),
-             {:ok, follower} <- User.maybe_direct_follow(follower, followed),
+             {:ok, follower, followed} <- User.maybe_direct_follow(follower, followed),
              {:ok, _, _, _} <- CommonAPI.follow(follower, followed) do
           followed
         else

lib/pleroma/user/search.ex

@@ -85,7 +85,6 @@ defmodule Pleroma.User.Search do
     |> base_query(following)
     |> filter_blocked_user(for_user)
     |> filter_invisible_users()
-    |> filter_discoverable_users()
     |> filter_internal_users()
     |> filter_blocked_domains(for_user)
     |> fts_search(query_string)
@@ -163,10 +162,6 @@ defmodule Pleroma.User.Search do
     from(q in query, where: q.invisible == false)
   end
 
-  defp filter_discoverable_users(query) do
-    from(q in query, where: q.is_discoverable == true)
-  end
-
   defp filter_internal_users(query) do
     from(q in query, where: q.actor_type != "Application")
   end

lib/pleroma/utils.ex

@@ -3,6 +3,14 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Utils do
+  @posix_error_codes ~w(
+    eacces eagain ebadf ebadmsg ebusy edeadlk edeadlock edquot eexist efault
+    efbig eftype eintr einval eio eisdir eloop emfile emlink emultihop
+    enametoolong enfile enobufs enodev enolck enolink enoent enomem enospc
+    enosr enostr enosys enotblk enotdir enotsup enxio eopnotsupp eoverflow
+    eperm epipe erange erofs espipe esrch estale etxtbsy exdev
+  )a
+
   def compile_dir(dir) when is_binary(dir) do
     dir
     |> File.ls!()
@@ -44,4 +52,12 @@ defmodule Pleroma.Utils do
       error -> error
     end
   end
+
+  @spec posix_error_message(atom()) :: binary()
+  def posix_error_message(code) when code in @posix_error_codes do
+    error_message = Gettext.dgettext(Pleroma.Web.Gettext, "posix_errors", "#{code}")
+    "(POSIX error: #{error_message})"
+  end
+
+  def posix_error_message(_), do: ""
 end

lib/pleroma/web.ex

@@ -20,6 +20,7 @@ defmodule Pleroma.Web do
   below.
   """
 
+  alias Pleroma.Helpers.AuthHelper
   alias Pleroma.Web.Plugs.EnsureAuthenticatedPlug
   alias Pleroma.Web.Plugs.EnsurePublicOrAuthenticatedPlug
   alias Pleroma.Web.Plugs.ExpectAuthenticatedCheckPlug
@@ -75,7 +76,7 @@ defmodule Pleroma.Web do
       defp maybe_drop_authentication_if_oauth_check_ignored(conn) do
         if PlugHelper.plug_called?(conn, ExpectPublicOrAuthenticatedCheckPlug) and
              not PlugHelper.plug_called_or_skipped?(conn, OAuthScopesPlug) do
-          OAuthScopesPlug.drop_auth_info(conn)
+          AuthHelper.drop_auth_info(conn)
         else
           conn
         end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -32,6 +32,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   require Logger
   require Pleroma.Constants
 
+  @behaviour Pleroma.Web.ActivityPub.ActivityPub.Persisting
+
   defp get_recipients(%{"type" => "Create"} = data) do
     to = Map.get(data, "to", [])
     cc = Map.get(data, "cc", [])
@@ -85,13 +87,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp increase_replies_count_if_reply(_create_data), do: :noop
 
   @object_types ~w[ChatMessage Question Answer Audio Video Event Article]
-  @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
+  @impl true
   def persist(%{"type" => type} = object, meta) when type in @object_types do
     with {:ok, object} <- Object.create(object) do
       {:ok, object, meta}
     end
   end
 
+  @impl true
   def persist(object, meta) do
     with local <- Keyword.fetch!(meta, :local),
          {recipients, _, _} <- get_recipients(object),
@@ -123,7 +126,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       # Splice in the child object if we have one.
       activity = Maps.put_if_present(activity, :object, object)
 
-      BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id})
+      ConcurrentLimiter.limit(Pleroma.Web.RichMedia.Helpers, fn ->
+        Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
+      end)
 
       {:ok, activity}
     else
@@ -332,7 +337,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   @spec flag(map()) :: {:ok, Activity.t()} | {:error, any()}
-  def flag(
+  def flag(params) do
+    with {:ok, result} <- Repo.transaction(fn -> do_flag(params) end) do
+      result
+    end
+  end
+
+  defp do_flag(
          %{
            actor: actor,
            context: _context,
@@ -358,7 +369,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
          {:ok, activity} <- insert(flag_data, local),
          {:ok, stripped_activity} <- strip_report_status_data(activity),
          _ <- notify_and_stream(activity),
-         :ok <- maybe_federate(stripped_activity) do
+         :ok <-
+           maybe_federate(stripped_activity) do
       User.all_superusers()
       |> Enum.filter(fn user -> not is_nil(user.email) end)
       |> Enum.each(fn superuser ->
@@ -368,6 +380,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       end)
 
       {:ok, activity}
+    else
+      {:error, error} -> Repo.rollback(error)
     end
   end
 
@@ -1289,12 +1303,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   def fetch_follow_information_for_user(user) do
     with {:ok, following_data} <-
-           Fetcher.fetch_and_contain_remote_object_from_id(user.following_address,
+           Fetcher.fetch_and_contain_remote_object_from_id(user.following_address),
-             force_http: true
-           ),
          {:ok, hide_follows} <- collection_private(following_data),
          {:ok, followers_data} <-
-           Fetcher.fetch_and_contain_remote_object_from_id(user.follower_address, force_http: true),
+           Fetcher.fetch_and_contain_remote_object_from_id(user.follower_address),
          {:ok, hide_followers} <- collection_private(followers_data) do
       {:ok,
        %{
@@ -1368,8 +1380,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  def fetch_and_prepare_user_from_ap_id(ap_id, opts \\ []) do
+  def fetch_and_prepare_user_from_ap_id(ap_id) do
-    with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id, opts),
+    with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id),
          {:ok, data} <- user_data_from_user_object(data) do
       {:ok, maybe_update_follow_information(data)}
     else
@@ -1412,13 +1424,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  def make_user_from_ap_id(ap_id, opts \\ []) do
+  def make_user_from_ap_id(ap_id) do
     user = User.get_cached_by_ap_id(ap_id)
 
     if user && !User.ap_enabled?(user) do
       Transmogrifier.upgrade_user_from_ap_id(ap_id)
     else
-      with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, opts) do
+      with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id) do
         if user do
           user
           |> User.remote_user_changeset(data)

lib/pleroma/web/activity_pub/activity_pub/persisting.ex

@@ -0,0 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ActivityPub.Persisting do
+  @callback persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
+end

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -82,7 +82,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   def object(conn, _) do
     with ap_id <- Endpoint.url() <> conn.request_path,
          %Object{} = object <- Object.get_cached_by_ap_id(ap_id),
-         {_, true} <- {:public?, Visibility.is_public?(object)} do
+         {_, true} <- {:public?, Visibility.is_public?(object)},
+         {_, false} <- {:local?, Visibility.is_local_public?(object)} do
       conn
       |> assign(:tracking_fun_data, object.id)
       |> set_cache_ttl_for(object)
@@ -92,6 +93,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     else
       {:public?, false} ->
         {:error, :not_found}
+
+      {:local?, true} ->
+        {:error, :not_found}
     end
   end
 
@@ -108,7 +112,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   def activity(conn, _params) do
     with ap_id <- Endpoint.url() <> conn.request_path,
          %Activity{} = activity <- Activity.normalize(ap_id),
-         {_, true} <- {:public?, Visibility.is_public?(activity)} do
+         {_, true} <- {:public?, Visibility.is_public?(activity)},
+         {_, false} <- {:local?, Visibility.is_local_public?(activity)} do
       conn
       |> maybe_set_tracking_data(activity)
       |> set_cache_ttl_for(activity)
@@ -117,6 +122,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
       |> render("object.json", object: activity)
     else
       {:public?, false} -> {:error, :not_found}
+      {:local?, true} -> {:error, :not_found}
       nil -> {:error, :not_found}
     end
   end

lib/pleroma/web/activity_pub/builder.ex

@@ -222,6 +222,9 @@ defmodule Pleroma.Web.ActivityPub.Builder do
         actor.ap_id == Relay.ap_id() ->
           [actor.follower_address]
 
+        public? and Visibility.is_local_public?(object) ->
+          [actor.follower_address, object.data["actor"], Pleroma.Constants.as_local_public()]
+
         public? ->
           [actor.follower_address, object.data["actor"], Pleroma.Constants.as_public()]
 

lib/pleroma/web/activity_pub/mrf.ex

@@ -5,6 +5,8 @@
 defmodule Pleroma.Web.ActivityPub.MRF do
   require Logger
 
+  @behaviour Pleroma.Web.ActivityPub.MRF.PipelineFiltering
+
   @mrf_config_descriptions [
     %{
       group: :pleroma,
@@ -70,6 +72,7 @@ defmodule Pleroma.Web.ActivityPub.MRF do
 
   def filter(%{} = object), do: get_policies() |> filter(object)
 
+  @impl true
   def pipeline_filter(%{} = message, meta) do
     object = meta[:object_data]
     ap_id = message["object"]

lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex

@@ -8,7 +8,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
 
   alias Pleroma.HTTP
   alias Pleroma.Web.MediaProxy
-  alias Pleroma.Workers.BackgroundWorker
 
   require Logger
 
@@ -17,7 +16,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
     recv_timeout: 10_000
   ]
 
-  def perform(:prefetch, url) do
+  defp prefetch(url) do
     # Fetching only proxiable resources
     if MediaProxy.enabled?() and MediaProxy.url_proxiable?(url) do
       # If preview proxy is enabled, it'll also hit media proxy (so we're caching both requests)
@@ -25,17 +24,25 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
 
       Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}")
 
-      HTTP.get(prefetch_url, [], @adapter_options)
+      if Pleroma.Config.get(:env) == :test do
+        fetch(prefetch_url)
+      else
+        ConcurrentLimiter.limit(MediaProxy, fn ->
+          Task.start(fn -> fetch(prefetch_url) end)
+        end)
+      end
     end
   end
 
-  def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do
+  defp fetch(url), do: HTTP.get(url, [], @adapter_options)
+
+  defp preload(%{"object" => %{"attachment" => attachments}} = _message) do
     Enum.each(attachments, fn
       %{"url" => url} when is_list(url) ->
         url
         |> Enum.each(fn
           %{"href" => href} ->
-            BackgroundWorker.enqueue("media_proxy_prefetch", %{"url" => href})
+            prefetch(href)
 
           x ->
             Logger.debug("Unhandled attachment URL object #{inspect(x)}")
@@ -51,7 +58,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
         %{"type" => "Create", "object" => %{"attachment" => attachments} = _object} = message
       )
       when is_list(attachments) and length(attachments) > 0 do
-    BackgroundWorker.enqueue("media_proxy_preload", %{"message" => message})
+    preload(message)
 
     {:ok, message}
   end

lib/pleroma/web/activity_pub/mrf/pipeline_filtering.ex

@@ -0,0 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.PipelineFiltering do
+  @callback pipeline_filter(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()}
+end

lib/pleroma/web/activity_pub/object_validator.ex

@@ -9,6 +9,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   the system.
   """
 
+  @behaviour Pleroma.Web.ActivityPub.ObjectValidator.Validating
+
   alias Pleroma.Activity
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Object
@@ -32,7 +34,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   alias Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.UpdateValidator
 
-  @spec validate(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()}
+  @impl true
   def validate(object, meta)
 
   def validate(%{"type" => type} = object, meta)

lib/pleroma/web/activity_pub/object_validator/validating.ex

@@ -0,0 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidator.Validating do
+  @callback validate(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()}
+end

lib/pleroma/web/activity_pub/object_validators/announce_validator.ex

@@ -67,7 +67,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator do
          %Object{} = object <- Object.get_cached_by_ap_id(object),
          false <- Visibility.is_public?(object) do
       same_actor = object.data["actor"] == actor.ap_id
-      is_public = Pleroma.Constants.as_public() in (get_field(cng, :to) ++ get_field(cng, :cc))
+      recipients = get_field(cng, :to) ++ get_field(cng, :cc)
+      local_public = Pleroma.Constants.as_local_public()
+
+      is_public =
+        Enum.member?(recipients, Pleroma.Constants.as_public()) or
+          Enum.member?(recipients, local_public)
 
       cond do
         same_actor && is_public ->

lib/pleroma/web/activity_pub/pipeline.ex

@@ -11,14 +11,22 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do
   alias Pleroma.Web.ActivityPub.MRF
   alias Pleroma.Web.ActivityPub.ObjectValidator
   alias Pleroma.Web.ActivityPub.SideEffects
+  alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.Federator
 
+  @side_effects Config.get([:pipeline, :side_effects], SideEffects)
+  @federator Config.get([:pipeline, :federator], Federator)
+  @object_validator Config.get([:pipeline, :object_validator], ObjectValidator)
+  @mrf Config.get([:pipeline, :mrf], MRF)
+  @activity_pub Config.get([:pipeline, :activity_pub], ActivityPub)
+  @config Config.get([:pipeline, :config], Config)
+
   @spec common_pipeline(map(), keyword()) ::
           {:ok, Activity.t() | Object.t(), keyword()} | {:error, any()}
   def common_pipeline(object, meta) do
     case Repo.transaction(fn -> do_common_pipeline(object, meta) end) do
       {:ok, {:ok, activity, meta}} ->
-        SideEffects.handle_after_transaction(meta)
+        @side_effects.handle_after_transaction(meta)
         {:ok, activity, meta}
 
       {:ok, value} ->
@@ -34,13 +42,13 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do
 
   def do_common_pipeline(object, meta) do
     with {_, {:ok, validated_object, meta}} <-
-           {:validate_object, ObjectValidator.validate(object, meta)},
+           {:validate_object, @object_validator.validate(object, meta)},
          {_, {:ok, mrfd_object, meta}} <-
-           {:mrf_object, MRF.pipeline_filter(validated_object, meta)},
+           {:mrf_object, @mrf.pipeline_filter(validated_object, meta)},
          {_, {:ok, activity, meta}} <-
-           {:persist_object, ActivityPub.persist(mrfd_object, meta)},
+           {:persist_object, @activity_pub.persist(mrfd_object, meta)},
          {_, {:ok, activity, meta}} <-
-           {:execute_side_effects, SideEffects.handle(activity, meta)},
+           {:execute_side_effects, @side_effects.handle(activity, meta)},
          {_, {:ok, _}} <- {:federation, maybe_federate(activity, meta)} do
       {:ok, activity, meta}
     else
@@ -53,9 +61,9 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do
 
   defp maybe_federate(%Activity{} = activity, meta) do
     with {:ok, local} <- Keyword.fetch(meta, :local) do
-      do_not_federate = meta[:do_not_federate] || !Config.get([:instance, :federating])
+      do_not_federate = meta[:do_not_federate] || !@config.get([:instance, :federating])
 
-      if !do_not_federate && local do
+      if !do_not_federate and local and not Visibility.is_local_public?(activity) do
         activity =
           if object = Keyword.get(meta, :object_data) do
             %{activity | data: Map.put(activity.data, "object", object)}
@@ -63,7 +71,7 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do
             activity
           end
 
-        Federator.publish(activity)
+        @federator.publish(activity)
         {:ok, :federated}
       else
         {:ok, :not_federated}

lib/pleroma/web/activity_pub/publisher.ex

@@ -13,7 +13,6 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Web.ActivityPub.Transmogrifier
-  alias Pleroma.Web.FedSockets
 
   require Pleroma.Constants
 
@@ -50,28 +49,6 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
   """
   def publish_one(%{inbox: inbox, json: json, actor: %User{} = actor, id: id} = params) do
     Logger.debug("Federating #{id} to #{inbox}")
-
-    case FedSockets.get_or_create_fed_socket(inbox) do
-      {:ok, fedsocket} ->
-        Logger.debug("publishing via fedsockets - #{inspect(inbox)}")
-        FedSockets.publish(fedsocket, json)
-
-      _ ->
-        Logger.debug("publishing via http - #{inspect(inbox)}")
-        http_publish(inbox, actor, json, params)
-    end
-  end
-
-  def publish_one(%{actor_id: actor_id} = params) do
-    actor = User.get_cached_by_id(actor_id)
-
-    params
-    |> Map.delete(:actor_id)
-    |> Map.put(:actor, actor)
-    |> publish_one()
-  end
-
-  defp http_publish(inbox, actor, json, params) do
     uri = %{path: path} = URI.parse(inbox)
     digest = "SHA-256=" <> (:crypto.hash(:sha256, json) |> Base.encode64())
 
@@ -110,6 +87,15 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
     end
   end
 
+  def publish_one(%{actor_id: actor_id} = params) do
+    actor = User.get_cached_by_id(actor_id)
+
+    params
+    |> Map.delete(:actor_id)
+    |> Map.put(:actor, actor)
+    |> publish_one()
+  end
+
   defp signature_host(%URI{port: port, scheme: scheme, host: host}) do
     if port == URI.default_port(scheme) do
       host

lib/pleroma/web/activity_pub/side_effects.ex

@@ -24,15 +24,20 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.Push
   alias Pleroma.Web.Streamer
-  alias Pleroma.Workers.BackgroundWorker
 
   require Logger
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
+  @behaviour Pleroma.Web.ActivityPub.SideEffects.Handling
+
+  @impl true
   def handle(object, meta \\ [])
 
   # Task this handles
   # - Follows
   # - Sends a notification
+  @impl true
   def handle(
         %{
           data: %{
@@ -48,10 +53,9 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
          %User{} = followed <- User.get_cached_by_ap_id(actor),
          %User{} = follower <- User.get_cached_by_ap_id(follower_id),
          {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "accept"),
-         {:ok, _relationship} <- FollowingRelationship.update(follower, followed, :follow_accept) do
+         {:ok, _follower, followed} <-
+           FollowingRelationship.update(follower, followed, :follow_accept) do
       Notification.update_notification_type(followed, follow_activity)
-      User.update_follower_count(followed)
-      User.update_following_count(follower)
     end
 
     {:ok, object, meta}
@@ -61,6 +65,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   # - Rejects all existing follow activities for this person
   # - Updates the follow state
   # - Dismisses notification
+  @impl true
   def handle(
         %{
           data: %{
@@ -87,6 +92,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   # - Follows if possible
   # - Sends a notification
   # - Generates accept or reject if appropriate
+  @impl true
   def handle(
         %{
           data: %{
@@ -100,7 +106,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
       ) do
     with %User{} = follower <- User.get_cached_by_ap_id(following_user),
          %User{} = followed <- User.get_cached_by_ap_id(followed_user),
-         {_, {:ok, _}, _, _} <-
+         {_, {:ok, _, _}, _, _} <-
            {:following, User.follow(follower, followed, :follow_pending), follower, followed} do
       if followed.local && !followed.is_locked do
         {:ok, accept_data, _} = Builder.accept(followed, object)
@@ -128,6 +134,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
 
   # Tasks this handles:
   # - Unfollow and block
+  @impl true
   def handle(
         %{data: %{"type" => "Block", "object" => blocked_user, "actor" => blocking_user}} =
           object,
@@ -146,6 +153,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   #
   # For a local user, we also get a changeset with the full information, so we
   # can update non-federating, non-activitypub settings as well.
+  @impl true
   def handle(%{data: %{"type" => "Update", "object" => updated_object}} = object, meta) do
     if changeset = Keyword.get(meta, :user_update_changeset) do
       changeset
@@ -164,6 +172,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   # Tasks this handles:
   # - Add like to object
   # - Set up notification
+  @impl true
   def handle(%{data: %{"type" => "Like"}} = object, meta) do
     liked_object = Object.get_by_ap_id(object.data["object"])
     Utils.add_like_to_object(object, liked_object)
@@ -181,6 +190,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   # - Increase replies count
   # - Set up ActivityExpiration
   # - Set up notifications
+  @impl true
   def handle(%{data: %{"type" => "Create"}} = activity, meta) do
     with {:ok, object, meta} <- handle_object_creation(meta[:object_data], meta),
          %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
@@ -191,7 +201,9 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
         Object.increase_replies_count(in_reply_to)
       end
 
-      BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id})
+      ConcurrentLimiter.limit(Pleroma.Web.RichMedia.Helpers, fn ->
+        Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
+      end)
 
       meta =
         meta
@@ -207,6 +219,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   # - Add announce to object
   # - Set up notification
   # - Stream out the announce
+  @impl true
   def handle(%{data: %{"type" => "Announce"}} = object, meta) do
     announced_object = Object.get_by_ap_id(object.data["object"])
     user = User.get_cached_by_ap_id(object.data["actor"])
@@ -224,6 +237,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
     {:ok, object, meta}
   end
 
+  @impl true
   def handle(%{data: %{"type" => "Undo", "object" => undone_object}} = object, meta) do
     with undone_object <- Activity.get_by_ap_id(undone_object),
          :ok <- handle_undoing(undone_object) do
@@ -234,6 +248,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   # Tasks this handles:
   # - Add reaction to object
   # - Set up notification
+  @impl true
   def handle(%{data: %{"type" => "EmojiReact"}} = object, meta) do
     reacted_object = Object.get_by_ap_id(object.data["object"])
     Utils.add_emoji_reaction_to_object(object, reacted_object)
@@ -250,6 +265,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   # - Reduce the user note count
   # - Reduce the reply count
   # - Stream out the activity
+  @impl true
   def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object, meta) do
     deleted_object =
       Object.normalize(deleted_object, false) ||
@@ -295,6 +311,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   end
 
   # Nothing to do
+  @impl true
   def handle(object, meta) do
     {:ok, object, meta}
   end
@@ -312,7 +329,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
             {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
             {:ok, cm_ref} = MessageReference.create(chat, object, user.ap_id != actor.ap_id)
 
-            Cachex.put(
+            @cachex.put(
               :chat_message_id_idempotency_key_cache,
               cm_ref.id,
               meta[:idempotency_key]
@@ -439,6 +456,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
     |> Keyword.put(:notifications, notifications ++ existing)
   end
 
+  @impl true
   def handle_after_transaction(meta) do
     meta
     |> send_notifications()

lib/pleroma/web/activity_pub/side_effects/handling.ex

@@ -0,0 +1,8 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.SideEffects.Handling do
+  @callback handle(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()}
+  @callback handle_after_transaction(map()) :: map()
+end

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -1008,7 +1008,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def upgrade_user_from_ap_id(ap_id) do
     with %User{local: false} = user <- User.get_cached_by_ap_id(ap_id),
-         {:ok, data} <- ActivityPub.fetch_and_prepare_user_from_ap_id(ap_id, force_http: true),
+         {:ok, data} <- ActivityPub.fetch_and_prepare_user_from_ap_id(ap_id),
          {:ok, user} <- update_user(user, data) do
       TransmogrifierWorker.enqueue("user_upgrade", %{"user_id" => user.id})
       {:ok, user}

lib/pleroma/web/activity_pub/utils.ex

@@ -175,7 +175,8 @@ defmodule Pleroma.Web.ActivityPub.Utils do
     outgoing_blocks = Config.get([:activitypub, :outgoing_blocks])
 
     with true <- Config.get!([:instance, :federating]),
-         true <- type != "Block" || outgoing_blocks do
+         true <- type != "Block" || outgoing_blocks,
+         false <- Visibility.is_local_public?(activity) do
       Pleroma.Web.Federator.publish(activity)
     end
 
@@ -701,14 +702,30 @@ defmodule Pleroma.Web.ActivityPub.Utils do
 
   def make_flag_data(_, _), do: %{}
 
-  defp build_flag_object(%{account: account, statuses: statuses} = _) do
+  defp build_flag_object(%{account: account, statuses: statuses}) do
-    [account.ap_id] ++ build_flag_object(%{statuses: statuses})
+    [account.ap_id | build_flag_object(%{statuses: statuses})]
   end
 
   defp build_flag_object(%{statuses: statuses}) do
     Enum.map(statuses || [], &build_flag_object/1)
   end
 
+  defp build_flag_object(%Activity{data: %{"id" => id}, object: %{data: data}}) do
+    activity_actor = User.get_by_ap_id(data["actor"])
+
+    %{
+      "type" => "Note",
+      "id" => id,
+      "content" => data["content"],
+      "published" => data["published"],
+      "actor" =>
+        AccountView.render(
+          "show.json",
+          %{user: activity_actor, skip_visibility_check: true}
+        )
+    }
+  end
+
   defp build_flag_object(act) when is_map(act) or is_binary(act) do
     id =
       case act do
@@ -719,24 +736,16 @@ defmodule Pleroma.Web.ActivityPub.Utils do
 
     case Activity.get_by_ap_id_with_object(id) do
       %Activity{} = activity ->
-        activity_actor = User.get_by_ap_id(activity.object.data["actor"])
+        build_flag_object(activity)
 
-        %{
+      nil ->
-          "type" => "Note",
+        if activity = Activity.get_by_object_ap_id_with_object(id) do
-          "id" => activity.data["id"],
+          build_flag_object(activity)
-          "content" => activity.object.data["content"],
+        else
-          "published" => activity.object.data["published"],
-          "actor" =>
-            AccountView.render(
-              "show.json",
-              %{user: activity_actor, skip_visibility_check: true}
-            )
-        }
-
-      _ ->
           %{"id" => id, "deleted" => true}
         end
     end
+  end
 
   defp build_flag_object(_), do: []
 

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -110,6 +110,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do
       "endpoints" => endpoints,
       "attachment" => fields,
       "tag" => emoji_tags,
+      # Note: key name is indeed "discoverable" (not an error)
       "discoverable" => user.is_discoverable,
       "capabilities" => capabilities
     }

lib/pleroma/web/activity_pub/visibility.ex

@@ -17,7 +17,19 @@ defmodule Pleroma.Web.ActivityPub.Visibility do
   def is_public?(%Activity{data: %{"type" => "Move"}}), do: true
   def is_public?(%Activity{data: data}), do: is_public?(data)
   def is_public?(%{"directMessage" => true}), do: false
-  def is_public?(data), do: Utils.label_in_message?(Pleroma.Constants.as_public(), data)
+
+  def is_public?(data) do
+    Utils.label_in_message?(Pleroma.Constants.as_public(), data) or
+      Utils.label_in_message?(Pleroma.Constants.as_local_public(), data)
+  end
+
+  def is_local_public?(%Object{data: data}), do: is_local_public?(data)
+  def is_local_public?(%Activity{data: data}), do: is_local_public?(data)
+
+  def is_local_public?(data) do
+    Utils.label_in_message?(Pleroma.Constants.as_local_public(), data) and
+      not Utils.label_in_message?(Pleroma.Constants.as_public(), data)
+  end
 
   def is_private?(activity) do
     with false <- is_public?(activity),
@@ -114,6 +126,9 @@ defmodule Pleroma.Web.ActivityPub.Visibility do
       Pleroma.Constants.as_public() in cc ->
         "unlisted"
 
+      Pleroma.Constants.as_local_public() in to ->
+        "local"
+
       # this should use the sql for the object's activity
       Enum.any?(to, &String.contains?(&1, "/followers")) ->
         "private"

lib/pleroma/web/admin_api/controllers/admin_api_controller.ex

@@ -415,7 +415,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   def confirm_email(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
     users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
 
-    User.toggle_confirmation(users)
+    User.confirm(users)
 
     ModerationLog.insert_log(%{actor: admin, subject: users, action: "confirm_email"})
 

lib/pleroma/web/admin_api/controllers/frontend_controller.ex

@@ -0,0 +1,40 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.FrontendController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Config
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
+
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+  plug(OAuthScopesPlug, %{scopes: ["write"], admin: true} when action == :install)
+  plug(OAuthScopesPlug, %{scopes: ["read"], admin: true} when action == :index)
+  action_fallback(Pleroma.Web.AdminAPI.FallbackController)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.FrontendOperation
+
+  def index(conn, _params) do
+    installed = installed()
+
+    frontends =
+      [:frontends, :available]
+      |> Config.get([])
+      |> Enum.map(fn {name, desc} ->
+        Map.put(desc, "installed", name in installed)
+      end)
+
+    render(conn, "index.json", frontends: frontends)
+  end
+
+  def install(%{body_params: params} = conn, _params) do
+    with :ok <- Pleroma.Frontend.install(params.name, Map.delete(params, :name)) do
+      index(conn, %{})
+    end
+  end
+
+  defp installed do
+    File.ls!(Pleroma.Frontend.dir())
+  end
+end

lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex

@@ -9,6 +9,8 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do
   alias Pleroma.Web.MediaProxy
   alias Pleroma.Web.Plugs.OAuthScopesPlug
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
   plug(Pleroma.Web.ApiSpec.CastAndValidate)
 
   plug(
@@ -38,7 +40,7 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do
 
   defp fetch_entries(params) do
     MediaProxy.cache_table()
-    |> Cachex.stream!(Cachex.Query.create(true, :key))
+    |> @cachex.stream!(Cachex.Query.create(true, :key))
     |> filter_entries(params[:query])
   end
 

lib/pleroma/web/admin_api/controllers/report_controller.ex

@@ -50,10 +50,13 @@ defmodule Pleroma.Web.AdminAPI.ReportController do
       Enum.map(reports, fn report ->
         case CommonAPI.update_report_state(report.id, report.state) do
           {:ok, activity} ->
+            report = Activity.get_by_id_with_user_actor(activity.id)
+
             ModerationLog.insert_log(%{
               action: "report_update",
               actor: admin,
-              subject: activity
+              subject: activity,
+              subject_actor: report.user_actor
             })
 
             activity
@@ -73,11 +76,13 @@ defmodule Pleroma.Web.AdminAPI.ReportController do
   def notes_create(%{assigns: %{user: user}, body_params: %{content: content}} = conn, %{
         id: report_id
       }) do
-    with {:ok, _} <- ReportNote.create(user.id, report_id, content) do
+    with {:ok, _} <- ReportNote.create(user.id, report_id, content),
+         report <- Activity.get_by_id_with_user_actor(report_id) do
       ModerationLog.insert_log(%{
         action: "report_note",
         actor: user,
-        subject: Activity.get_by_id(report_id),
+        subject: report,
+        subject_actor: report.user_actor,
         text: content
       })
 
@@ -91,11 +96,13 @@ defmodule Pleroma.Web.AdminAPI.ReportController do
         id: note_id,
         report_id: report_id
       }) do
-    with {:ok, note} <- ReportNote.destroy(note_id) do
+    with {:ok, note} <- ReportNote.destroy(note_id),
+         report <- Activity.get_by_id_with_user_actor(report_id) do
       ModerationLog.insert_log(%{
         action: "report_note_delete",
         actor: user,
-        subject: Activity.get_by_id(report_id),
+        subject: report,
+        subject_actor: report.user_actor,
         text: note.content
       })
 

lib/pleroma/web/admin_api/views/frontend_view.ex

@@ -0,0 +1,21 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.FrontendView do
+  use Pleroma.Web, :view
+
+  def render("index.json", %{frontends: frontends}) do
+    render_many(frontends, __MODULE__, "show.json")
+  end
+
+  def render("show.json", %{frontend: frontend}) do
+    %{
+      name: frontend["name"],
+      git: frontend["git"],
+      build_url: frontend["build_url"],
+      ref: frontend["ref"],
+      installed: frontend["installed"]
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/account_operation.ex

@@ -139,6 +139,12 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
             :query,
             %Schema{type: :array, items: VisibilityScope},
             "Exclude visibilities"
+          ),
+          Operation.parameter(
+            :with_muted,
+            :query,
+            BooleanLike,
+            "Include reactions from muted acccounts."
           )
         ] ++ pagination_params(),
       responses: %{
@@ -618,7 +624,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
           allOf: [BooleanLike],
           nullable: true,
           description:
-            "Discovery of this account in search results and other services is allowed."
+            "Discovery (listing, indexing) of this account by external services (search bots etc.) is allowed."
         },
         actor_type: ActorType
       },

lib/pleroma/web/api_spec/operations/admin/frontend_operation.ex

@@ -0,0 +1,85 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.FrontendOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def index_operation do
+    %Operation{
+      tags: ["Admin", "Reports"],
+      summary: "Get a list of available frontends",
+      operationId: "AdminAPI.FrontendController.index",
+      security: [%{"oAuth" => ["read"]}],
+      responses: %{
+        200 => Operation.response("Response", "application/json", list_of_frontends()),
+        403 => Operation.response("Forbidden", "application/json", ApiError)
+      }
+    }
+  end
+
+  def install_operation do
+    %Operation{
+      tags: ["Admin", "Reports"],
+      summary: "Install a frontend",
+      operationId: "AdminAPI.FrontendController.install",
+      security: [%{"oAuth" => ["read"]}],
+      requestBody: request_body("Parameters", install_request(), required: true),
+      responses: %{
+        200 => Operation.response("Response", "application/json", list_of_frontends()),
+        403 => Operation.response("Forbidden", "application/json", ApiError),
+        400 => Operation.response("Error", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp list_of_frontends do
+    %Schema{
+      type: :array,
+      items: %Schema{
+        type: :object,
+        properties: %{
+          name: %Schema{type: :string},
+          git: %Schema{type: :string, format: :uri, nullable: true},
+          build_url: %Schema{type: :string, format: :uri, nullable: true},
+          ref: %Schema{type: :string},
+          installed: %Schema{type: :boolean}
+        }
+      }
+    }
+  end
+
+  defp install_request do
+    %Schema{
+      title: "FrontendInstallRequest",
+      type: :object,
+      required: [:name],
+      properties: %{
+        name: %Schema{
+          type: :string
+        },
+        ref: %Schema{
+          type: :string
+        },
+        file: %Schema{
+          type: :string
+        },
+        build_url: %Schema{
+          type: :string
+        },
+        build_dir: %Schema{
+          type: :string
+        }
+      }
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/emoji_reaction_operation.ex

@@ -24,6 +24,12 @@ defmodule Pleroma.Web.ApiSpec.EmojiReactionOperation do
         Operation.parameter(:id, :path, FlakeID, "Status ID", required: true),
         Operation.parameter(:emoji, :path, :string, "Filter by a single unicode emoji",
           required: nil
+        ),
+        Operation.parameter(
+          :with_muted,
+          :query,
+          :boolean,
+          "Include reactions from muted acccounts."
         )
       ],
       security: [%{"oAuth" => ["read:statuses"]}],

lib/pleroma/web/api_spec/operations/pleroma_emoji_file_operation.ex

@@ -27,7 +27,8 @@ defmodule Pleroma.Web.ApiSpec.PleromaEmojiFileOperation do
         422 => Operation.response("Unprocessable Entity", "application/json", ApiError),
         404 => Operation.response("Not Found", "application/json", ApiError),
         400 => Operation.response("Bad Request", "application/json", ApiError),
-        409 => Operation.response("Conflict", "application/json", ApiError)
+        409 => Operation.response("Conflict", "application/json", ApiError),
+        500 => Operation.response("Error", "application/json", ApiError)
       }
     }
   end

lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex

@@ -169,7 +169,8 @@ defmodule Pleroma.Web.ApiSpec.PleromaEmojiPackOperation do
       responses: %{
         200 => ok_response(),
         400 => Operation.response("Bad Request", "application/json", ApiError),
-        404 => Operation.response("Not Found", "application/json", ApiError)
+        404 => Operation.response("Not Found", "application/json", ApiError),
+        500 => Operation.response("Error", "application/json", ApiError)
       }
     }
   end
@@ -184,7 +185,8 @@ defmodule Pleroma.Web.ApiSpec.PleromaEmojiPackOperation do
       parameters: [name_param()],
       responses: %{
         200 => Operation.response("Metadata", "application/json", metadata()),
-        400 => Operation.response("Bad Request", "application/json", ApiError)
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        500 => Operation.response("Error", "application/json", ApiError)
       }
     }
   end

lib/pleroma/web/api_spec/operations/status_operation.ex

@@ -31,6 +31,12 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do
           :query,
           %Schema{type: :array, items: FlakeID},
           "Array of status IDs"
+        ),
+        Operation.parameter(
+          :with_muted,
+          :query,
+          BooleanLike,
+          "Include reactions from muted acccounts."
         )
       ],
       operationId: "StatusController.index",
@@ -67,7 +73,15 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do
       description: "View information about a status",
       operationId: "StatusController.show",
       security: [%{"oAuth" => ["read:statuses"]}],
-      parameters: [id_param()],
+      parameters: [
+        id_param(),
+        Operation.parameter(
+          :with_muted,
+          :query,
+          BooleanLike,
+          "Include reactions from muted acccounts."
+        )
+      ],
       responses: %{
         200 => status_response(),
         404 => Operation.response("Not Found", "application/json", ApiError)

lib/pleroma/web/api_spec/operations/subscription_operation.ex

@@ -146,6 +146,11 @@ defmodule Pleroma.Web.ApiSpec.SubscriptionOperation do
                   allOf: [BooleanLike],
                   nullable: true,
                   description: "Receive chat notifications?"
+                },
+                "pleroma:emoji_reaction": %Schema{
+                  allOf: [BooleanLike],
+                  nullable: true,
+                  description: "Receive emoji reaction notifications?"
                 }
               }
             }
@@ -210,6 +215,16 @@ defmodule Pleroma.Web.ApiSpec.SubscriptionOperation do
                   allOf: [BooleanLike],
                   nullable: true,
                   description: "Receive poll notifications?"
+                },
+                "pleroma:chat_mention": %Schema{
+                  allOf: [BooleanLike],
+                  nullable: true,
+                  description: "Receive chat notifications?"
+                },
+                "pleroma:emoji_reaction": %Schema{
+                  allOf: [BooleanLike],
+                  nullable: true,
+                  description: "Receive emoji reaction notifications?"
                 }
               }
             }

lib/pleroma/web/api_spec/schemas/account.ex

@@ -127,7 +127,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do
               discoverable: %Schema{
                 type: :boolean,
                 description:
-                  "whether the user allows discovery of the account in search results and other services."
+                  "whether the user allows indexing / listing of the account by external services (search engines etc.)."
               },
               no_rich_text: %Schema{
                 type: :boolean,

lib/pleroma/web/api_spec/schemas/visibility_scope.ex

@@ -9,6 +9,6 @@ defmodule Pleroma.Web.ApiSpec.Schemas.VisibilityScope do
     title: "VisibilityScope",
     description: "Status visibility",
     type: :string,
-    enum: ["public", "unlisted", "private", "direct", "list"]
+    enum: ["public", "unlisted", "local", "private", "direct", "list"]
   })
 end

lib/pleroma/web/common_api.ex

@@ -15,6 +15,7 @@ defmodule Pleroma.Web.CommonAPI do
   alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
+  alias Pleroma.Web.CommonAPI.ActivityDraft
 
   import Pleroma.Web.Gettext
   import Pleroma.Web.CommonAPI.Utils
@@ -358,7 +359,7 @@ defmodule Pleroma.Web.CommonAPI do
   def get_visibility(_, _, %Participation{}), do: {"direct", "direct"}
 
   def get_visibility(%{visibility: visibility}, in_reply_to, _)
-      when visibility in ~w{public unlisted private direct},
+      when visibility in ~w{public local unlisted private direct},
       do: {visibility, get_replied_to_visibility(in_reply_to)}
 
   def get_visibility(%{visibility: "list:" <> list_id}, in_reply_to, _) do
@@ -399,31 +400,13 @@ defmodule Pleroma.Web.CommonAPI do
   end
 
   def listen(user, data) do
-    visibility = Map.get(data, :visibility, "public")
+    with {:ok, draft} <- ActivityDraft.listen(user, data) do
-
+      ActivityPub.listen(draft.changes)
-    with {to, cc} <- get_to_and_cc(user, [], nil, visibility, nil),
-         listen_data <-
-           data
-           |> Map.take([:album, :artist, :title, :length])
-           |> Map.new(fn {key, value} -> {to_string(key), value} end)
-           |> Map.put("type", "Audio")
-           |> Map.put("to", to)
-           |> Map.put("cc", cc)
-           |> Map.put("actor", user.ap_id),
-         {:ok, activity} <-
-           ActivityPub.listen(%{
-             actor: user,
-             to: to,
-             object: listen_data,
-             context: Utils.generate_context_id(),
-             additional: %{"cc" => cc}
-           }) do
-      {:ok, activity}
     end
   end
 
   def post(user, %{status: _} = data) do
-    with {:ok, draft} <- Pleroma.Web.CommonAPI.ActivityDraft.create(user, data) do
+    with {:ok, draft} <- ActivityDraft.create(user, data) do
       ActivityPub.create(draft.changes, draft.preview?)
     end
   end

lib/pleroma/web/common_api/activity_draft.ex

@@ -22,7 +22,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
             in_reply_to_conversation: nil,
             visibility: nil,
             expires_at: nil,
-            poll: nil,
+            extra: nil,
             emoji: %{},
             content_html: nil,
             mentions: [],
@@ -35,9 +35,14 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
             preview?: false,
             changes: %{}
 
-  def create(user, params) do
+  def new(user, params) do
     %__MODULE__{user: user}
     |> put_params(params)
+  end
+
+  def create(user, params) do
+    user
+    |> new(params)
     |> status()
     |> summary()
     |> with_valid(&attachments/1)
@@ -57,6 +62,30 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
     |> validate()
   end
 
+  def listen(user, params) do
+    user
+    |> new(params)
+    |> visibility()
+    |> to_and_cc()
+    |> context()
+    |> listen_object()
+    |> with_valid(&changes/1)
+    |> validate()
+  end
+
+  defp listen_object(draft) do
+    object =
+      draft.params
+      |> Map.take([:album, :artist, :title, :length])
+      |> Map.new(fn {key, value} -> {to_string(key), value} end)
+      |> Map.put("type", "Audio")
+      |> Map.put("to", draft.to)
+      |> Map.put("cc", draft.cc)
+      |> Map.put("actor", draft.user.ap_id)
+
+    %__MODULE__{draft | object: object}
+  end
+
   defp put_params(draft, params) do
     params = Map.put_new(params, :in_reply_to_status_id, params[:in_reply_to_id])
     %__MODULE__{draft | params: params}
@@ -121,7 +150,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
   defp poll(draft) do
     case Utils.make_poll_data(draft.params) do
       {:ok, {poll, poll_emoji}} ->
-        %__MODULE__{draft | poll: poll, emoji: Map.merge(draft.emoji, poll_emoji)}
+        %__MODULE__{draft | extra: poll, emoji: Map.merge(draft.emoji, poll_emoji)}
 
       {:error, message} ->
         add_error(draft, message)
@@ -129,32 +158,18 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
   end
 
   defp content(draft) do
-    {content_html, mentions, tags} =
+    {content_html, mentioned_users, tags} = Utils.make_content_html(draft)
-      Utils.make_content_html(
+
-        draft.status,
+    mentions =
-        draft.attachments,
+      mentioned_users
-        draft.params,
+      |> Enum.map(fn {_, mentioned_user} -> mentioned_user.ap_id end)
-        draft.visibility
+      |> Utils.get_addressed_users(draft.params[:to])
-      )
 
     %__MODULE__{draft | content_html: content_html, mentions: mentions, tags: tags}
   end
 
   defp to_and_cc(draft) do
-    addressed_users =
+    {to, cc} = Utils.get_to_and_cc(draft)
-      draft.mentions
-      |> Enum.map(fn {_, mentioned_user} -> mentioned_user.ap_id end)
-      |> Utils.get_addressed_users(draft.params[:to])
-
-    {to, cc} =
-      Utils.get_to_and_cc(
-        draft.user,
-        addressed_users,
-        draft.in_reply_to,
-        draft.visibility,
-        draft.in_reply_to_conversation
-      )
-
     %__MODULE__{draft | to: to, cc: cc}
   end
 
@@ -172,19 +187,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
     emoji = Map.merge(Pleroma.Emoji.Formatter.get_emoji_map(draft.full_payload), draft.emoji)
 
     object =
-      Utils.make_note_data(
+      Utils.make_note_data(draft)
-        draft.user.ap_id,
-        draft.to,
-        draft.context,
-        draft.content_html,
-        draft.attachments,
-        draft.in_reply_to,
-        draft.tags,
-        draft.summary,
-        draft.cc,
-        draft.sensitive,
-        draft.poll
-      )
       |> Map.put("emoji", emoji)
       |> Map.put("source", draft.status)
 

lib/pleroma/web/common_api/utils.ex

@@ -16,6 +16,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
+  alias Pleroma.Web.CommonAPI.ActivityDraft
   alias Pleroma.Web.MediaProxy
   alias Pleroma.Web.Plugs.AuthenticationPlug
 
@@ -50,67 +51,62 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     {_, descs} = Jason.decode(descs_str)
 
     Enum.map(ids, fn media_id ->
-      case Repo.get(Object, media_id) do
+      with %Object{data: data} <- Repo.get(Object, media_id) do
-        %Object{data: data} ->
         Map.put(data, "name", descs[media_id])
-
-        _ ->
-          nil
       end
     end)
     |> Enum.reject(&is_nil/1)
   end
 
-  @spec get_to_and_cc(
+  @spec get_to_and_cc(ActivityDraft.t()) :: {list(String.t()), list(String.t())}
-          User.t(),
-          list(String.t()),
-          Activity.t() | nil,
-          String.t(),
-          Participation.t() | nil
-        ) :: {list(String.t()), list(String.t())}
 
-  def get_to_and_cc(_, _, _, _, %Participation{} = participation) do
+  def get_to_and_cc(%{in_reply_to_conversation: %Participation{} = participation}) do
     participation = Repo.preload(participation, :recipients)
     {Enum.map(participation.recipients, & &1.ap_id), []}
   end
 
-  def get_to_and_cc(user, mentioned_users, inReplyTo, "public", _) do
+  def get_to_and_cc(%{visibility: visibility} = draft) when visibility in ["public", "local"] do
-    to = [Pleroma.Constants.as_public() | mentioned_users]
+    to =
-    cc = [user.follower_address]
+      case visibility do
+        "public" -> [Pleroma.Constants.as_public() | draft.mentions]
+        "local" -> [Pleroma.Constants.as_local_public() | draft.mentions]
+      end
 
-    if inReplyTo do
+    cc = [draft.user.follower_address]
-      {Enum.uniq([inReplyTo.data["actor"] | to]), cc}
+
+    if draft.in_reply_to do
+      {Enum.uniq([draft.in_reply_to.data["actor"] | to]), cc}
     else
       {to, cc}
     end
   end
 
-  def get_to_and_cc(user, mentioned_users, inReplyTo, "unlisted", _) do
+  def get_to_and_cc(%{visibility: "unlisted"} = draft) do
-    to = [user.follower_address | mentioned_users]
+    to = [draft.user.follower_address | draft.mentions]
     cc = [Pleroma.Constants.as_public()]
 
-    if inReplyTo do
+    if draft.in_reply_to do
-      {Enum.uniq([inReplyTo.data["actor"] | to]), cc}
+      {Enum.uniq([draft.in_reply_to.data["actor"] | to]), cc}
     else
       {to, cc}
     end
   end
 
-  def get_to_and_cc(user, mentioned_users, inReplyTo, "private", _) do
+  def get_to_and_cc(%{visibility: "private"} = draft) do
-    {to, cc} = get_to_and_cc(user, mentioned_users, inReplyTo, "direct", nil)
+    {to, cc} = get_to_and_cc(struct(draft, visibility: "direct"))
-    {[user.follower_address | to], cc}
+    {[draft.user.follower_address | to], cc}
   end
 
-  def get_to_and_cc(_user, mentioned_users, inReplyTo, "direct", _) do
+  def get_to_and_cc(%{visibility: "direct"} = draft) do
     # If the OP is a DM already, add the implicit actor.
-    if inReplyTo && Visibility.is_direct?(inReplyTo) do
+    if draft.in_reply_to && Visibility.is_direct?(draft.in_reply_to) do
-      {Enum.uniq([inReplyTo.data["actor"] | mentioned_users]), []}
+      {Enum.uniq([draft.in_reply_to.data["actor"] | draft.mentions]), []}
     else
-      {mentioned_users, []}
+      {draft.mentions, []}
     end
   end
 
-  def get_to_and_cc(_user, mentions, _inReplyTo, {:list, _}, _), do: {mentions, []}
+  def get_to_and_cc(%{visibility: {:list, _}, mentions: mentions}), do: {mentions, []}
 
   def get_addressed_users(_, to) when is_list(to) do
     User.get_ap_ids_by_nicknames(to)
@@ -203,30 +199,25 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     end
   end
 
-  def make_content_html(
+  def make_content_html(%ActivityDraft{} = draft) do
-        status,
-        attachments,
-        data,
-        visibility
-      ) do
     attachment_links =
-      data
+      draft.params
       |> Map.get("attachment_links", Config.get([:instance, :attachment_links]))
       |> truthy_param?()
 
-    content_type = get_content_type(data[:content_type])
+    content_type = get_content_type(draft.params[:content_type])
 
     options =
-      if visibility == "direct" && Config.get([:instance, :safe_dm_mentions]) do
+      if draft.visibility == "direct" && Config.get([:instance, :safe_dm_mentions]) do
         [safe_mention: true]
       else
         []
       end
 
-    status
+    draft.status
     |> format_input(content_type, options)
-    |> maybe_add_attachments(attachments, attachment_links)
+    |> maybe_add_attachments(draft.attachments, attachment_links)
-    |> maybe_add_nsfw_tag(data)
+    |> maybe_add_nsfw_tag(draft.params)
   end
 
   defp get_content_type(content_type) do
@@ -308,33 +299,21 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     |> Formatter.html_escape("text/html")
   end
 
-  def make_note_data(
+  def make_note_data(%ActivityDraft{} = draft) do
-        actor,
-        to,
-        context,
-        content_html,
-        attachments,
-        in_reply_to,
-        tags,
-        summary \\ nil,
-        cc \\ [],
-        sensitive \\ false,
-        extra_params \\ %{}
-      ) do
     %{
       "type" => "Note",
-      "to" => to,
+      "to" => draft.to,
-      "cc" => cc,
+      "cc" => draft.cc,
-      "content" => content_html,
+      "content" => draft.content_html,
-      "summary" => summary,
+      "summary" => draft.summary,
-      "sensitive" => truthy_param?(sensitive),
+      "sensitive" => draft.sensitive,
-      "context" => context,
+      "context" => draft.context,
-      "attachment" => attachments,
+      "attachment" => draft.attachments,
-      "actor" => actor,
+      "actor" => draft.user.ap_id,
-      "tag" => Keyword.values(tags) |> Enum.uniq()
+      "tag" => Keyword.values(draft.tags) |> Enum.uniq()
     }
-    |> add_in_reply_to(in_reply_to)
+    |> add_in_reply_to(draft.in_reply_to)
-    |> Map.merge(extra_params)
+    |> Map.merge(draft.extra)
   end
 
   defp add_in_reply_to(object, nil), do: object