From c12879841836035da0a4dc2117459801f16a27fb Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 17 Nov 2021 22:29:49 +0300 Subject: [PATCH] Rename Activity.Search to Search.DatabaseSearch --- config/config.exs | 3 +- config/test.exs | 2 +- docs/configuration/search.md | 2 +- lib/pleroma/activity.ex | 2 +- lib/pleroma/activity/search.ex | 153 ----------------- lib/pleroma/search.ex | 24 +++ lib/pleroma/search/database_search.ex | 157 ++++++++++++++++-- lib/pleroma/search/meilisearch.ex | 2 +- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- lib/pleroma/web/activity_pub/side_effects.ex | 4 +- .../controllers/search_controller.ex | 2 +- .../database_search_test.ex} | 10 +- 12 files changed, 182 insertions(+), 181 deletions(-) delete mode 100644 lib/pleroma/activity/search.ex create mode 100644 lib/pleroma/search.ex rename test/pleroma/{activity/search_test.exs => search/database_search_test.ex} (81%) diff --git a/config/config.exs b/config/config.exs index c2b4ca996..731804503 100644 --- a/config/config.exs +++ b/config/config.exs @@ -856,7 +856,8 @@ config :pleroma, ConcurrentLimiter, [ config :pleroma, :search, provider: Pleroma.Search.Builtin -config :pleroma, Pleroma.Search, module: Pleroma.Activity.Search +config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch + config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", private_key: nil # Import environment specific config. This must remain at the bottom diff --git a/config/test.exs b/config/test.exs index 4d8d73502..7fbababdf 100644 --- a/config/test.exs +++ b/config/test.exs @@ -134,7 +134,7 @@ config :pleroma, :side_effects, ap_streamer: Pleroma.Web.ActivityPub.ActivityPubMock, logger: Pleroma.LoggerMock -config :pleroma, Pleroma.Search, module: Pleroma.Activity +config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", private_key: nil diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 9adc7884f..c7e77d9c2 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -6,7 +6,7 @@ To use built-in search that has no external dependencies, set the search module to `Pleroma.Activity`: -> config :pleroma, Pleroma.Search, module: Pleroma.Activity +> config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch While it has no external dependencies, it has problems with performance and relevancy. diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 4106feef6..abfe778d2 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -367,7 +367,7 @@ defmodule Pleroma.Activity do from(activity in query, where: activity.actor not in subquery(deactivated_users_query)) end - defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search + defdelegate search(user, query, options \\ []), to: Pleroma.Search.DatabaseSearch def direct_conversation_id(activity, for_user) do alias Pleroma.Conversation.Participation diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex deleted file mode 100644 index 8352ba20a..000000000 --- a/lib/pleroma/activity/search.ex +++ /dev/null @@ -1,153 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Activity.Search do - alias Pleroma.Activity - alias Pleroma.Object.Fetcher - alias Pleroma.Pagination - alias Pleroma.User - alias Pleroma.Web.ActivityPub.Visibility - - require Pleroma.Constants - - import Ecto.Query - - def search(user, search_query, options \\ []) do - index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin - limit = Enum.min([Keyword.get(options, :limit), 40]) - offset = Keyword.get(options, :offset, 0) - author = Keyword.get(options, :author) - - search_function = - if :persistent_term.get({Pleroma.Repo, :postgres_version}) >= 11 do - :websearch - else - :plain - end - - try do - Activity - |> Activity.with_preloaded_object() - |> Activity.restrict_deactivated_users() - |> restrict_public() - |> query_with(index_type, search_query, search_function) - |> maybe_restrict_local(user) - |> maybe_restrict_author(author) - |> maybe_restrict_blocked(user) - |> Pagination.fetch_paginated( - %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum}, - :offset - ) - |> maybe_fetch(user, search_query) - rescue - _ -> maybe_fetch([], user, search_query) - end - end - - def add_to_index(_activity), do: nil - def remove_from_index(_object), do: nil - - def maybe_restrict_author(query, %User{} = author) do - Activity.Queries.by_author(query, author) - end - - def maybe_restrict_author(query, _), do: query - - def maybe_restrict_blocked(query, %User{} = user) do - Activity.Queries.exclude_authors(query, User.blocked_users_ap_ids(user)) - end - - def maybe_restrict_blocked(query, _), do: query - - def restrict_public(q) do - from([a, o] in q, - where: fragment("?->>'type' = 'Create'", a.data), - where: ^Pleroma.Constants.as_public() in a.recipients - ) - end - - defp query_with(q, :gin, search_query, :plain) do - %{rows: [[tsc]]} = - Ecto.Adapters.SQL.query!( - Pleroma.Repo, - "select current_setting('default_text_search_config')::regconfig::oid;" - ) - - from([a, o] in q, - where: - fragment( - "to_tsvector(?::oid::regconfig, ?->>'content') @@ plainto_tsquery(?)", - ^tsc, - o.data, - ^search_query - ) - ) - end - - defp query_with(q, :gin, search_query, :websearch) do - %{rows: [[tsc]]} = - Ecto.Adapters.SQL.query!( - Pleroma.Repo, - "select current_setting('default_text_search_config')::regconfig::oid;" - ) - - from([a, o] in q, - where: - fragment( - "to_tsvector(?::oid::regconfig, ?->>'content') @@ websearch_to_tsquery(?)", - ^tsc, - o.data, - ^search_query - ) - ) - end - - defp query_with(q, :rum, search_query, :plain) do - from([a, o] in q, - where: - fragment( - "? @@ plainto_tsquery(?)", - o.fts_content, - ^search_query - ), - order_by: [fragment("? <=> now()::date", o.inserted_at)] - ) - end - - defp query_with(q, :rum, search_query, :websearch) do - from([a, o] in q, - where: - fragment( - "? @@ websearch_to_tsquery(?)", - o.fts_content, - ^search_query - ), - order_by: [fragment("? <=> now()::date", o.inserted_at)] - ) - end - - def maybe_restrict_local(q, user) do - limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated) - - case {limit, user} do - {:all, _} -> restrict_local(q) - {:unauthenticated, %User{}} -> q - {:unauthenticated, _} -> restrict_local(q) - {false, _} -> q - end - end - - defp restrict_local(q), do: where(q, local: true) - - def maybe_fetch(activities, user, search_query) do - with true <- Regex.match?(~r/https?:/, search_query), - {:ok, object} <- Fetcher.fetch_object_from_id(search_query), - %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), - true <- Visibility.visible_for_user?(activity, user) do - [activity | activities] - else - _ -> activities - end - end -end diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex new file mode 100644 index 000000000..ae0b28c54 --- /dev/null +++ b/lib/pleroma/search.ex @@ -0,0 +1,24 @@ +defmodule Pleroma.Search do + def add_to_index(activity) do + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.add_to_index(activity) end) + end) + end + + def remove_from_index(object) do + # Also delete from search index + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.remove_from_index(object) end) + end) + end + + def search(query, options) do + search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) + + search_module.search(options[:for_user], query, options) + end +end diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index be0e19be0..5a8b8ca67 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -1,24 +1,153 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Search.DatabaseSearch do - def add_to_index(activity) do - search_module = Pleroma.Config.get([Pleroma.Search, :module]) + alias Pleroma.Activity + alias Pleroma.Object.Fetcher + alias Pleroma.Pagination + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Visibility - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.add_to_index(activity) end) - end) + require Pleroma.Constants + + import Ecto.Query + + def search(user, search_query, options \\ []) do + index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin + limit = Enum.min([Keyword.get(options, :limit), 40]) + offset = Keyword.get(options, :offset, 0) + author = Keyword.get(options, :author) + + search_function = + if :persistent_term.get({Pleroma.Repo, :postgres_version}) >= 11 do + :websearch + else + :plain + end + + try do + Activity + |> Activity.with_preloaded_object() + |> Activity.restrict_deactivated_users() + |> restrict_public() + |> query_with(index_type, search_query, search_function) + |> maybe_restrict_local(user) + |> maybe_restrict_author(author) + |> maybe_restrict_blocked(user) + |> Pagination.fetch_paginated( + %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum}, + :offset + ) + |> maybe_fetch(user, search_query) + rescue + _ -> maybe_fetch([], user, search_query) + end end - def remove_from_index(object) do - # Also delete from search index - search_module = Pleroma.Config.get([Pleroma.Search, :module]) + def add_to_index(_activity), do: nil + def remove_from_index(_object), do: nil - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.remove_from_index(object) end) - end) + def maybe_restrict_author(query, %User{} = author) do + Activity.Queries.by_author(query, author) end - def search(query, options) do - search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) + def maybe_restrict_author(query, _), do: query - search_module.search(options[:for_user], query, options) + def maybe_restrict_blocked(query, %User{} = user) do + Activity.Queries.exclude_authors(query, User.blocked_users_ap_ids(user)) + end + + def maybe_restrict_blocked(query, _), do: query + + def restrict_public(q) do + from([a, o] in q, + where: fragment("?->>'type' = 'Create'", a.data), + where: ^Pleroma.Constants.as_public() in a.recipients + ) + end + + defp query_with(q, :gin, search_query, :plain) do + %{rows: [[tsc]]} = + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + "select current_setting('default_text_search_config')::regconfig::oid;" + ) + + from([a, o] in q, + where: + fragment( + "to_tsvector(?::oid::regconfig, ?->>'content') @@ plainto_tsquery(?)", + ^tsc, + o.data, + ^search_query + ) + ) + end + + defp query_with(q, :gin, search_query, :websearch) do + %{rows: [[tsc]]} = + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + "select current_setting('default_text_search_config')::regconfig::oid;" + ) + + from([a, o] in q, + where: + fragment( + "to_tsvector(?::oid::regconfig, ?->>'content') @@ websearch_to_tsquery(?)", + ^tsc, + o.data, + ^search_query + ) + ) + end + + defp query_with(q, :rum, search_query, :plain) do + from([a, o] in q, + where: + fragment( + "? @@ plainto_tsquery(?)", + o.fts_content, + ^search_query + ), + order_by: [fragment("? <=> now()::date", o.inserted_at)] + ) + end + + defp query_with(q, :rum, search_query, :websearch) do + from([a, o] in q, + where: + fragment( + "? @@ websearch_to_tsquery(?)", + o.fts_content, + ^search_query + ), + order_by: [fragment("? <=> now()::date", o.inserted_at)] + ) + end + + def maybe_restrict_local(q, user) do + limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated) + + case {limit, user} do + {:all, _} -> restrict_local(q) + {:unauthenticated, %User{}} -> q + {:unauthenticated, _} -> restrict_local(q) + {false, _} -> q + end + end + + defp restrict_local(q), do: where(q, local: true) + + def maybe_fetch(activities, user, search_query) do + with true <- Regex.match?(~r/https?:/, search_query), + {:ok, object} <- Fetcher.fetch_object_from_id(search_query), + %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), + true <- Visibility.visible_for_user?(activity, user) do + [activity | activities] + else + _ -> activities + end end end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 41f99ad9f..fa9e27b03 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Search.Meilisearch do alias Pleroma.Activity - import Pleroma.Activity.Search + import Pleroma.Search.DatabaseSearch import Ecto.Query defp meili_headers do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 92b1be62c..e6548a818 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -141,7 +141,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end) # Add local posts to search index - if local, do: Pleroma.Search.DatabaseSearch.add_to_index(activity) + if local, do: Pleroma.Search.add_to_index(activity) {:ok, activity} else diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index d08d6aa70..097afa30e 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -223,7 +223,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) end) - Pleroma.Search.DatabaseSearch.add_to_index(Map.put(activity, :object, object)) + Pleroma.Search.add_to_index(Map.put(activity, :object, object)) meta = meta @@ -325,7 +325,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do if result == :ok do Notification.create_notifications(object) - Pleroma.Search.DatabaseSearch.remove_from_index(deleted_object) + Pleroma.Search.remove_from_index(deleted_object) {:ok, object, meta} else diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 05fa4144d..751d46cdf 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -76,7 +76,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do end defp resource_search(_, "statuses", query, options) do - statuses = with_fallback(fn -> Pleroma.Search.DatabaseSearch.search(query, options) end) + statuses = with_fallback(fn -> Pleroma.Search.search(query, options) end) StatusView.render("index.json", activities: statuses, diff --git a/test/pleroma/activity/search_test.exs b/test/pleroma/search/database_search_test.ex similarity index 81% rename from test/pleroma/activity/search_test.exs rename to test/pleroma/search/database_search_test.ex index 657fbc627..2387ac29b 100644 --- a/test/pleroma/activity/search_test.exs +++ b/test/pleroma/search/database_search_test.ex @@ -2,8 +2,8 @@ # Copyright © 2017-2021 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Activity.SearchTest do - alias Pleroma.Activity.Search +defmodule Pleroma.Search.DatabaseSearchTest do + alias Pleroma.Search.DatabaseSearch alias Pleroma.Web.CommonAPI import Pleroma.Factory @@ -13,7 +13,7 @@ defmodule Pleroma.Activity.SearchTest do user = insert(:user) {:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"}) - [result] = Search.search(nil, "wednesday") + [result] = DatabaseSearch.search(nil, "wednesday") assert result.id == post.id end @@ -28,7 +28,7 @@ defmodule Pleroma.Activity.SearchTest do {:ok, _post2} = CommonAPI.post(user, %{status: "it's wednesday my bros"}) # plainto doesn't understand complex queries - assert [result] = Search.search(nil, "wednesday -dudes") + assert [result] = DatabaseSearch.search(nil, "wednesday -dudes") assert result.id == post.id end @@ -38,7 +38,7 @@ defmodule Pleroma.Activity.SearchTest do {:ok, _post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"}) {:ok, other_post} = CommonAPI.post(user, %{status: "it's wednesday my bros"}) - assert [result] = Search.search(nil, "wednesday -dudes") + assert [result] = DatabaseSearch.search(nil, "wednesday -dudes") assert result.id == other_post.id end