Merge pull request 'Preserve Meilisearch’s result ranking' (#772) from Oneric/akkoma:search-meili-order into develop

Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/772

CHANGELOG.md

@@ -14,6 +14,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## Added
 - Implement [FEP-67ff](https://codeberg.org/fediverse/fep/src/branch/main/fep/67ff/fep-67ff.md) (federation documentation)
 
+## Added
+- Meilisearch: it is now possible to use separate keys for search and admin actions
+
+## Fixed
+- Meilisearch: order of results returned from our REST API now actually matches how Meilisearch ranks results
+
 ## 2024.04
 
 ## Added

docs/docs/configuration/search.md

@@ -33,6 +33,7 @@ indexes faster when it can process many posts in a single batch.
 > config :pleroma, Pleroma.Search.Meilisearch,
 >    url: "http://127.0.0.1:7700/",
 >    private_key: "private key",
+>    search_key: "search key",
 >    initial_indexing_chunk_size: 100_000
 
 Information about setting up meilisearch can be found in the
@@ -45,7 +46,7 @@ is hardly usable on a somewhat big instance.
 ### Private key authentication (optional)
 
 To set the private key, use the `MEILI_MASTER_KEY` environment variable when starting. After setting the _master key_,
-you have to get the _private key_, which is actually used for authentication.
+you have to get the _private key_ and possibly _search key_, which are actually used for authentication.
 
 === "OTP"
     ```sh
@@ -57,7 +58,11 @@ you have to get the _private key_, which is actually used for authentication.
     mix pleroma.search.meilisearch show-keys <your master key here>
     ```
 
-You will see a "Default Admin API Key", this is the key you actually put into your configuration file.
+You will see a "Default Admin API Key", this is the key you actually put into
+your configuration file as `private_key`. You should also see a
+"Default Search API key", put this into your config as `search_key`.
+If your version of Meilisearch only showed the former,
+just leave `search_key` completely unset in Akkoma's config.
 
 ### Initial indexing
 

lib/mix/tasks/pleroma/search/meilisearch.ex

@@ -126,8 +126,12 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
     decoded = Jason.decode!(result.body)
 
     if decoded["results"] do
-      Enum.each(decoded["results"], fn %{"description" => desc, "key" => key} ->
+      Enum.each(decoded["results"], fn
-        IO.puts("#{desc}: #{key}")
+        %{"name" => name, "key" => key} ->
+          IO.puts("#{name}: #{key}")
+
+        %{"description" => desc, "key" => key} ->
+          IO.puts("#{desc}: #{key}")
       end)
     else
       IO.puts("Error fetching the keys, check the master key is correct: #{inspect(decoded)}")

lib/pleroma/activity.ex

@@ -258,6 +258,27 @@ defmodule Pleroma.Activity do
 
   def get_create_by_object_ap_id(_), do: nil
 
+  @doc """
+  Accepts a list of `ap__id`.
+  Returns a query yielding Create activities for the given objects,
+  in the same order as they were specified in the input list.
+  """
+  @spec get_presorted_create_by_object_ap_id([String.t()]) :: Ecto.Queryable.t()
+  def get_presorted_create_by_object_ap_id(ap_ids) do
+    from(
+      a in Activity,
+      join:
+        ids in fragment(
+          "SELECT * FROM UNNEST(?::text[]) WITH ORDINALITY AS ids(ap_id, ord)",
+          ^ap_ids
+        ),
+      on:
+        ids.ap_id == fragment("?->>'object'", a.data) and
+          fragment("?->>'type'", a.data) == "Create",
+      order_by: [asc: ids.ord]
+    )
+  end
+
   @doc """
   Accepts `ap_id` or list of `ap_id`.
   Returns a query.

lib/pleroma/search/meilisearch.ex

@@ -5,15 +5,27 @@ defmodule Pleroma.Search.Meilisearch do
   alias Pleroma.Activity
 
   import Pleroma.Search.DatabaseSearch
-  import Ecto.Query
 
   @behaviour Pleroma.Search.SearchBackend
 
-  defp meili_headers do
+  defp meili_headers(key) do
-    private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key])
+    key_header =
+      if is_nil(key), do: [], else: [{"Authorization", "Bearer #{key}"}]
 
-    [{"Content-Type", "application/json"}] ++
+    [{"Content-Type", "application/json"} | key_header]
-      if is_nil(private_key), do: [], else: [{"Authorization", "Bearer #{private_key}"}]
+  end
+
+  defp meili_headers_admin do
+    private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key])
+    meili_headers(private_key)
+  end
+
+  defp meili_headers_search do
+    search_key =
+      Pleroma.Config.get([Pleroma.Search.Meilisearch, :search_key]) ||
+        Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key])
+
+    meili_headers(search_key)
   end
 
   def meili_get(path) do
@@ -22,7 +34,7 @@ defmodule Pleroma.Search.Meilisearch do
     result =
       Pleroma.HTTP.get(
         Path.join(endpoint, path),
-        meili_headers()
+        meili_headers_admin()
       )
 
     with {:ok, res} <- result do
@@ -30,14 +42,14 @@ defmodule Pleroma.Search.Meilisearch do
     end
   end
 
-  def meili_post(path, params) do
+  defp meili_search(params) do
     endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
 
     result =
       Pleroma.HTTP.post(
-        Path.join(endpoint, path),
+        Path.join(endpoint, "/indexes/objects/search"),
         Jason.encode!(params),
-        meili_headers()
+        meili_headers_search()
       )
 
     with {:ok, res} <- result do
@@ -53,7 +65,7 @@ defmodule Pleroma.Search.Meilisearch do
         :put,
         Path.join(endpoint, path),
         Jason.encode!(params),
-        meili_headers(),
+        meili_headers_admin(),
         []
       )
 
@@ -70,7 +82,7 @@ defmodule Pleroma.Search.Meilisearch do
         :delete,
         Path.join(endpoint, path),
         "",
-        meili_headers(),
+        meili_headers_admin(),
         []
       )
   end
@@ -81,25 +93,20 @@ defmodule Pleroma.Search.Meilisearch do
     author = Keyword.get(options, :author)
 
     res =
-      meili_post(
+      meili_search(%{q: query, offset: offset, limit: limit})
-        "/indexes/objects/search",
-        %{q: query, offset: offset, limit: limit}
-      )
 
     with {:ok, result} <- res do
       hits = result["hits"] |> Enum.map(& &1["ap"])
 
       try do
         hits
-        |> Activity.create_by_object_ap_id()
+        |> Activity.get_presorted_create_by_object_ap_id()
-        |> Activity.with_preloaded_object()
         |> Activity.with_preloaded_object()
         |> Activity.restrict_deactivated_users()
         |> maybe_restrict_local(user)
         |> maybe_restrict_author(author)
         |> maybe_restrict_blocked(user)
         |> maybe_fetch(user, query)
-        |> order_by([object: obj], desc: obj.data["published"])
         |> Pleroma.Repo.all()
       rescue
         _ -> maybe_fetch([], user, query)

test/pleroma/activity_test.exs

@@ -41,6 +41,26 @@ defmodule Pleroma.ActivityTest do
     assert activity == found_activity
   end
 
+  test "returns activities by object's AP id in requested presorted order" do
+    a1 = insert(:note_activity)
+    o1 = Object.normalize(a1, fetch: false).data["id"]
+
+    a2 = insert(:note_activity)
+    o2 = Object.normalize(a2, fetch: false).data["id"]
+
+    a3 = insert(:note_activity)
+    o3 = Object.normalize(a3, fetch: false).data["id"]
+
+    a4 = insert(:note_activity)
+    o4 = Object.normalize(a4, fetch: false).data["id"]
+
+    found_activities =
+      Activity.get_presorted_create_by_object_ap_id([o3, o2, o4, o1])
+      |> Repo.all()
+
+    assert found_activities == [a3, a2, a4, a1]
+  end
+
   test "preloading a bookmark" do
     user = insert(:user)
     user2 = insert(:user)