Merge remote-tracking branch 'pleroma/develop' into nsfw-api-mrf

2021-06-17 15:38:10 -05:00 · 2021-06-17 15:38:10 -05:00 · 3a03d9b65f
commit 3a03d9b65f
parent a704d5499c a8adc300d4
356 changed files with 7959 additions and 2641 deletions
--- a/.gitignore
+++ b/.gitignore
@ -28,6 +28,7 @@ erl_crash.dump
 # variables.
 /config/*.secret.exs
 /config/generated_config.exs
+/config/runtime.exs
 /config/*.env


--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -8,7 +8,9 @@ variables: &global_variables
  MIX_ENV: test

 cache: &global_cache_policy
-  key: ${CI_COMMIT_REF_SLUG}
+  key:
+    files:
+      - mix.lock
  paths:
    - deps
    - _build
@ -22,20 +24,34 @@ stages:
  - docker

 before_script:
+  - echo $MIX_ENV
+  - rm -rf _build/*/lib/pleroma
  - apt-get update && apt-get install -y cmake
  - mix local.hex --force
  - mix local.rebar --force
+  - mix deps.get
  - apt-get -qq update
  - apt-get install -y libmagic-dev

+after_script:
+  - rm -rf _build/*/lib/pleroma
+
 build:
  stage: build
+  only:
+    changes:
+      - "**/*.ex"
+      - "**/*.exs"
+      - "mix.lock"
  script:
-  - mix deps.get
  - mix compile --force

 spec-build:
  stage: test
+  only:
+    changes:
+      - "lib/pleroma/web/api_spec/**/*.ex"
+      - "lib/pleroma/web/api_spec.ex"
  artifacts:
    paths:
    - spec.json
@ -52,13 +68,17 @@ benchmark:
    alias: postgres
    command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
  script:
-    - mix deps.get
    - mix ecto.create
    - mix ecto.migrate
    - mix pleroma.load_testing

 unit-testing:
  stage: test
+  only:
+    changes:
+      - "**/*.ex"
+      - "**/*.exs"
+      - "mix.lock"
  retry: 2
  cache: &testing_cache_policy
    <<: *global_cache_policy
@ -70,7 +90,6 @@ unit-testing:
    command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
  script:
    - apt-get update && apt-get install -y libimage-exiftool-perl ffmpeg
-    - mix deps.get
    - mix ecto.create
    - mix ecto.migrate
    - mix coveralls --preload-modules
@ -93,6 +112,11 @@ unit-testing:

 unit-testing-rum:
  stage: test
+  only:
+    changes:
+      - "**/*.ex"
+      - "**/*.exs"
+      - "mix.lock"
  retry: 2
  cache: *testing_cache_policy
  services:
@ -104,7 +128,6 @@ unit-testing-rum:
    RUM_ENABLED: "true"
  script:
    - apt-get update && apt-get install -y libimage-exiftool-perl ffmpeg
-    - mix deps.get
    - mix ecto.create
    - mix ecto.migrate
    - "mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
@ -112,17 +135,40 @@ unit-testing-rum:

 lint:
  stage: test
+  only:
+    changes:
+      - "**/*.ex"
+      - "**/*.exs"
+      - "mix.lock"
  cache: *testing_cache_policy
  script:
    - mix format --check-formatted

 analysis:
  stage: test
+  only:
+    changes:
+      - "**/*.ex"
+      - "**/*.exs"
+      - "mix.lock"
  cache: *testing_cache_policy
  script:
-    - mix deps.get
    - mix credo --strict --only=warnings,todo,fixme,consistency,readability

+cycles:
+  stage: test
+  image: elixir:1.11
+  only:
+    changes:
+      - "**/*.ex"
+      - "**/*.exs"
+      - "mix.lock"
+  cache: {}
+  script:
+    - mix deps.get
+    - mix compile
+    - mix xref graph --format cycles --label compile | awk '{print $0} END{exit ($0 != "No cycles found")}'
+
 docs-deploy:
  stage: deploy
  cache: *testing_cache_policy
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -4,6 +4,45 @@ All notable changes to this project will be documented in this file.

 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).

+## Unreleased
+
+### Changed
+
+- **Breaking:** Configuration: `:chat, enabled` moved to `:shout, enabled` and `:instance, chat_limit` moved to `:shout, limit`
+- Support for Erlang/OTP 24
+- The `application` metadata returned with statuses is no longer hardcoded. Apps that want to display these details will now have valid data for new posts after this change.
+- HTTPSecurityPlug now sends a response header to opt out of Google's FLoC (Federated Learning of Cohorts) targeted advertising.
+- Email address is now returned if requesting user is the owner of the user account so it can be exposed in client and FE user settings UIs.
+- Improved Twittercard and OpenGraph meta tag generation including thumbnails and image dimension metadata when available.
+
+### Added
+
+- MRF (`FollowBotPolicy`): New MRF Policy which makes a designated local Bot account attempt to follow all users in public Notes received by your instance. Users who require approving follower requests or have #nobot in their profile are excluded.
+- Return OAuth token `id` (primary key) in POST `/oauth/token`.
+- `AnalyzeMetadata` upload filter for extracting image/video attachment dimensions and generating blurhashes for images. Blurhashes for videos are not generated at this time.
+- Attachment dimensions and blurhashes are federated when available.
+- Pinned posts federation
+
+### Fixed
+- Don't crash so hard when email settings are invalid.
+- Checking activated Upload Filters for required commands.
+- Mix task `pleroma.database prune_objects`
+
+### Removed
+- **Breaking**: Remove deprecated `/api/qvitter/statuses/notifications/read` (replaced by `/api/v1/pleroma/notifications/read`)
+
+## Unreleased (Patch)
+
+### Fixed
+
+- Try to save exported ConfigDB settings (migrate_from_db) in the system temp directory if default location is not writable.
+- Uploading custom instance thumbnail via AdminAPI/AdminFE generated invalid URL to the image
+- Applying ConcurrentLimiter settings via AdminAPI
+- User login failures if their `notification_settings` were in a NULL state.
+- Mix task `pleroma.user delete_activities` query transaction timeout is now :infinity
+- MRF (`SimplePolicy`): Embedded objects are now checked. If any embedded object would be rejected, its parent is rejected. This fixes Announces leaking posts from blocked domains.
+- Fixed some Markdown issues, including trailing slash in links.
+
 ## [2.3.0] - 2020-03-01

 ### Security
@ -18,6 +57,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).

 - **Breaking**: Changed `mix pleroma.user toggle_confirmed` to `mix pleroma.user confirm`
 - **Breaking**: Changed `mix pleroma.user toggle_activated` to `mix pleroma.user activate/deactivate`
+- **Breaking:** NSFW hashtag is no longer added on sensitive posts
 - Polls now always return a `voters_count`, even if they are single-choice.
 - Admin Emails: The ap id is used as the user link in emails now.
 - Improved registration workflow for email confirmation and account approval modes.
@ -44,6 +84,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Pleroma API: Reroute `/api/pleroma/*` to `/api/v1/pleroma/*`

 </details>
+- Improved hashtag timeline performance (requires a background migration).

 ### Added

@ -67,6 +108,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 <details>
  <summary>API Changes</summary>
 - Admin API: (`GET /api/pleroma/admin/users`) filter users by `unconfirmed` status and `actor_type`.
+- Admin API: OpenAPI spec for the user-related operations
 - Pleroma API: `GET /api/v2/pleroma/chats` added. It is exactly like `GET /api/v1/pleroma/chats` except supports pagination.
 - Pleroma API: Add `idempotency_key` to the chat message entity that can be used for optimistic message sending.
 - Pleroma API: (`GET /api/v1/pleroma/federation_status`) Add a way to get a list of unreachable instances.
@ -498,7 +540,6 @@ switched to a new configuration mechanism, however it was not officially removed
 - Static-FE: Fix remote posts not being sanitized

 ### Fixed
-=======
 - Rate limiter crashes when there is no explicitly specified ip in the config
 - 500 errors when no `Accept` header is present if Static-FE is enabled
 - Instance panel not being updated immediately due to wrong `Cache-Control` headers
--- a/2
+++ b/2
@ -33,7 +33,7 @@ ARG DATA=/var/lib/pleroma

 RUN echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories &&\
 	apk update &&\
-	apk add exiftool imagemagick libmagic ncurses postgresql-client &&\
+	apk add exiftool ffmpeg imagemagick libmagic ncurses postgresql-client &&\
 	adduser --system --shell /bin/false --home ${HOME} pleroma &&\
 	mkdir -p ${DATA}/uploads &&\
 	mkdir -p ${DATA}/static &&\
--- a/README.md
+++ b/README.md
@ -35,6 +35,9 @@ Currently Pleroma is not packaged by any OS/Distros, but if you want to package
 ### Docker
 While we don’t provide docker files, other people have written very good ones. Take a look at <https://github.com/angristan/docker-pleroma> or <https://glitch.sh/sn0w/pleroma-docker>.

+### Raspberry Pi
+Community maintained Raspberry Pi image that you can flash and run Pleroma on your Raspberry Pi. Available here <https://github.com/guysoft/PleromaPi>.
+
 ### Compilation Troubleshooting
 If you ever encounter compilation issues during the updating of Pleroma, you can try these commands and see if they fix things:

@ -50,5 +53,5 @@ If you are not developing Pleroma, it is better to use the OTP release, which co
 - Latest Git revision: <https://docs-develop.pleroma.social>

 ## Community Channels
-* IRC: **#pleroma** and **#pleroma-dev** on freenode, webchat is available at <https://irc.pleroma.social>
-* Matrix: <https://matrix.to/#/#freenode_#pleroma:matrix.org> and <https://matrix.to/#/#freenode_#pleroma-dev:matrix.org>
+* IRC: **#pleroma** and **#pleroma-dev** on libera.chat, webchat is available at <https://irc.pleroma.social>
+* Matrix: [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat) and [#pleroma-dev:libera.chat](https://matrix.to/#/#pleroma-dev:libera.chat)
--- a/benchmarks/load_testing/activities.ex
+++ b/benchmarks/load_testing/activities.ex
@ -299,7 +299,7 @@ defmodule Pleroma.LoadTesting.Activities do
      "url" => [
        %{
          "href" =>
-            "#{Pleroma.Web.base_url()}/media/b1b873552422a07bf53af01f3c231c841db4dfc42c35efde681abaf0f2a4eab7.jpg",
+            "#{Pleroma.Web.Endpoint.url()}/media/b1b873552422a07bf53af01f3c231c841db4dfc42c35efde681abaf0f2a4eab7.jpg",
          "mediaType" => "image/jpeg",
          "type" => "Link"
        }
--- a/config/benchmark.exs
+++ b/config/benchmark.exs
@ -1,4 +1,4 @@
-use Mix.Config
+import Config

 # We don't run a server during test. If one is required,
 # you can enable the server option below.
--- a/config/config.exs
+++ b/config/config.exs
@ -41,7 +41,7 @@
 #
 # This configuration file is loaded before any dependency and
 # is restricted to this project.
-use Mix.Config
+import Config

 # General application configuration
 config :pleroma, ecto_repos: [Pleroma.Repo]
@ -190,7 +190,6 @@ config :pleroma, :instance,
  instance_thumbnail: "/instance/thumbnail.jpeg",
  limit: 5_000,
  description_limit: 5_000,
-  chat_limit: 5_000,
  remote_limit: 100_000,
  upload_limit: 16_000_000,
  avatar_upload_limit: 2_000_000,
@ -391,6 +390,11 @@ config :pleroma, :mrf_keyword,
  federated_timeline_removal: [],
  replace: []

+config :pleroma, :mrf_hashtag,
+  sensitive: ["nsfw"],
+  reject: [],
+  federated_timeline_removal: []
+
 config :pleroma, :mrf_subchain, match_actor: %{}

 config :pleroma, :mrf_activity_expiration, days: 365
@ -411,6 +415,8 @@ config :pleroma, :mrf_nsfw_api,
  unlist: false,
  reject: false

+config :pleroma, :mrf_follow_bot, follower_nickname: nil
+
 config :pleroma, :rich_media,
  enabled: true,
  ignore_hosts: [],
@ -457,7 +463,9 @@ config :pleroma, :media_preview_proxy,
  image_quality: 85,
  min_content_length: 100 * 1024

-config :pleroma, :chat, enabled: true
+config :pleroma, :shout,
+  enabled: true,
+  limit: 5_000

 config :phoenix, :format_encoders, json: Jason

@ -661,6 +669,10 @@ config :pleroma, :oauth2,

 config :pleroma, :database, rum_enabled: false

+config :pleroma, :features, improved_hashtag_timeline: :auto
+
+config :pleroma, :populate_hashtags_table, fault_rate_allowance: 0.01
+
 config :pleroma, :env, Mix.env()

 config :http_signatures,
--- a/config/description.exs
+++ b/config/description.exs
@ -1,4 +1,4 @@
-use Mix.Config
+import Config

 websocket_config = [
  path: "/websocket",
@ -461,6 +461,42 @@ config :pleroma, :config_description, [
  },
  %{
    group: :pleroma,
+    key: :features,
+    type: :group,
+    description: "Customizable features",
+    children: [
+      %{
+        key: :improved_hashtag_timeline,
+        type: {:dropdown, :atom},
+        description:
+          "Setting to force toggle / force disable improved hashtags timeline. `:enabled` forces hashtags to be fetched from `hashtags` table for hashtags timeline. `:disabled` forces object-embedded hashtags to be used (slower). Keep it `:auto` for automatic behaviour (it is auto-set to `:enabled` [unless overridden] when HashtagsTableMigrator completes).",
+        suggestions: [:auto, :enabled, :disabled]
+      }
+    ]
+  },
+  %{
+    group: :pleroma,
+    key: :populate_hashtags_table,
+    type: :group,
+    description: "`populate_hashtags_table` background migration settings",
+    children: [
+      %{
+        key: :fault_rate_allowance,
+        type: :float,
+        description:
+          "Max accepted rate of objects that failed in the migration. Any value from 0.0 which tolerates no errors to 1.0 which will enable the feature even if hashtags transfer failed for all records.",
+        suggestions: [0.01]
+      },
+      %{
+        key: :sleep_interval_ms,
+        type: :integer,
+        description:
+          "Sleep interval between each chunk of processed records in order to decrease the load on the system (defaults to 0 and should be keep default on most instances)."
+      }
+    ]
+  },
+  %{
+    group: :pleroma,
    key: :instance,
    type: :group,
    description: "Instance-related settings",
@ -509,14 +545,6 @@ config :pleroma, :config_description, [
        ]
      },
      %{
-        key: :chat_limit,
-        type: :integer,
-        description: "Character limit of the instance chat messages",
-        suggestions: [
-          5_000
-        ]
-      },
-      %{
        key: :remote_limit,
        type: :integer,
        description: "Hard character limit beyond which remote posts will be dropped",
@ -646,7 +674,8 @@ config :pleroma, :config_description, [
      %{
        key: :allow_relay,
        type: :boolean,
-        description: "Enable Pleroma's Relay, which makes it possible to follow a whole instance"
+        description:
+          "Permits remote instances to subscribe to all public posts of your instance. (Important!) This may increase the visibility of your instance."
      },
      %{
        key: :public,
@ -1146,7 +1175,6 @@ config :pleroma, :config_description, [
            alwaysShowSubjectInput: true,
            background: "/static/aurora_borealis.jpg",
            collapseMessageWithSubject: false,
-            disableChat: false,
            greentext: false,
            hideFilteredStatuses: false,
            hideMutedPosts: false,
@ -1194,12 +1222,6 @@ config :pleroma, :config_description, [
              "When a message has a subject (aka Content Warning), collapse it by default"
          },
          %{
-            key: :disableChat,
-            label: "PleromaFE Chat",
-            type: :boolean,
-            description: "Disables PleromaFE Chat component"
-          },
-          %{
            key: :greentext,
            label: "Greentext",
            type: :boolean,
@ -2616,13 +2638,22 @@ config :pleroma, :config_description, [
  },
  %{
    group: :pleroma,
-    key: :chat,
+    key: :shout,
    type: :group,
-    description: "Pleroma chat settings",
+    description: "Pleroma shout settings",
    children: [
      %{
        key: :enabled,
-        type: :boolean
+        type: :boolean,
+        description: "Enables the backend Shoutbox chat feature."
+      },
+      %{
+        key: :limit,
+        type: :integer,
+        description: "Shout message character limit.",
+        suggestions: [
+          5_000
+        ]
      }
    ]
  },
@ -2908,6 +2939,23 @@ config :pleroma, :config_description, [
  },
  %{
    group: :pleroma,
+    key: :mrf_follow_bot,
+    tab: :mrf,
+    related_policy: "Pleroma.Web.ActivityPub.MRF.FollowBotPolicy",
+    label: "MRF FollowBot Policy",
+    type: :group,
+    description: "Automatically follows newly discovered accounts.",
+    children: [
+      %{
+        key: :follower_nickname,
+        type: :string,
+        description: "The name of the bot account to use for following newly discovered users.",
+        suggestions: ["followbot"]
+      }
+    ]
+  },
+  %{
+    group: :pleroma,
    key: :modules,
    type: :group,
    description: "Custom Runtime Modules",
--- a/config/dev.exs
+++ b/config/dev.exs
@ -1,4 +1,4 @@
-use Mix.Config
+import Config

 # For development, we disable any cache and enable
 # debugging and code reloading.
@ -54,6 +54,10 @@ config :pleroma, Pleroma.Repo,

 config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: true

+# Reduce recompilation time
+# https://dashbit.co/blog/speeding-up-re-compilation-of-elixir-projects
+config :phoenix, :plug_init_mode, :runtime
+
 if File.exists?("./config/dev.secret.exs") do
  import_config "dev.secret.exs"
 else
--- a/config/dokku.exs
+++ b/config/dokku.exs
@ -1,4 +1,4 @@
-use Mix.Config
+import Config

 config :pleroma, Pleroma.Web.Endpoint,
  http: [
--- a/config/prod.exs
+++ b/config/prod.exs
@ -1,4 +1,4 @@
-use Mix.Config
+import Config

 # For production, we often load configuration from external
 # sources, such as your system environment. For this reason,
@ -63,7 +63,12 @@ config :logger, :ex_syslogger, level: :info

 # Finally import the config/prod.secret.exs
 # which should be versioned separately.
+if File.exists?("./config/prod.secret.exs") do
  import_config "prod.secret.exs"
+else
+  "`config/prod.secret.exs` not found. You may want to create one by running `mix pleroma.instance gen`"
+  |> IO.warn([])
+end

 if File.exists?("./config/prod.exported_from_db.secret.exs"),
  do: import_config("prod.exported_from_db.secret.exs")
--- a/config/test.exs
+++ b/config/test.exs
@ -1,4 +1,4 @@
-use Mix.Config
+import Config

 # We don't run a server during test. If one is required,
 # you can enable the server option below.
@ -133,6 +133,10 @@ config :pleroma, :side_effects,
  ap_streamer: Pleroma.Web.ActivityPub.ActivityPubMock,
  logger: Pleroma.LoggerMock

+# Reduce recompilation time
+# https://dashbit.co/blog/speeding-up-re-compilation-of-elixir-projects
+config :phoenix, :plug_init_mode, :runtime
+
 if File.exists?("./config/test.secret.exs") do
  import_config "test.secret.exs"
 else
--- a/docs/administration/CLI_tasks/config.md
+++ b/docs/administration/CLI_tasks/config.md
@ -32,16 +32,20 @@
    config :pleroma, configurable_from_database: false
    ```

-To delete transferred settings from database optional flag `-d` can be used. `<env>` is `prod` by default.
+Options:
+
+- `<path>` - where to save migrated config. E.g. `--path=/tmp`. If file saved into non standart folder, you must manually copy file into directory where Pleroma can read it. For OTP install path will be `PLEROMA_CONFIG_PATH` or `/etc/pleroma`. For installation from source - `config` directory in the pleroma folder.
+- `<env>` - environment, for which is migrated config. By default is `prod`.
+- To delete transferred settings from database optional flag `-d` can be used

 === "OTP"
    ```sh
-     ./bin/pleroma_ctl config migrate_from_db [--env=<env>] [-d]
+     ./bin/pleroma_ctl config migrate_from_db [--env=<env>] [-d] [--path=<path>]
    ```

 === "From Source"
    ```sh
-    mix pleroma.config migrate_from_db [--env=<env>] [-d]
+    mix pleroma.config migrate_from_db [--env=<env>] [-d] [--path=<path>]
    ```

 ## Dump all of the config settings defined in the database
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@ -8,9 +8,10 @@ For from source installations Pleroma configuration works by first importing the

 To add configuration to your config file, you can copy it from the base config. The latest version of it can be viewed [here](https://git.pleroma.social/pleroma/pleroma/blob/develop/config/config.exs). You can also use this file if you don't know how an option is supposed to be formatted.

-## :chat
+## :shout

-* `enabled` - Enables the backend chat. Defaults to `true`.
+* `enabled` - Enables the backend Shoutbox chat feature. Defaults to `true`.
+* `limit` - Shout character limit. Defaults to `5_000`

 ## :instance
 * `name`: The instance’s name.
@ -19,7 +20,6 @@ To add configuration to your config file, you can copy it from the base config.
 * `description`: The instance’s description, can be seen in nodeinfo and ``/api/v1/instance``.
 * `limit`: Posts character limit (CW/Subject included in the counter).
 * `description_limit`: The character limit for image descriptions.
-* `chat_limit`: Character limit of the instance chat messages.
 * `remote_limit`: Hard character limit beyond which remote posts will be dropped.
 * `upload_limit`: File size limit of uploads (except for avatar, background, banner).
 * `avatar_upload_limit`: File size limit of user’s profile avatars.
@ -37,7 +37,7 @@ To add configuration to your config file, you can copy it from the base config.
 * `federating`: Enable federation with other instances.
 * `federation_incoming_replies_max_depth`: Max. depth of reply-to activities fetching on incoming federation, to prevent out-of-memory situations while fetching very long threads. If set to `nil`, threads of any depth will be fetched. Lower this value if you experience out-of-memory crashes.
 * `federation_reachability_timeout_days`: Timeout (in days) of each external federation target being unreachable prior to pausing federating to it.
-* `allow_relay`: Enable Pleroma’s Relay, which makes it possible to follow a whole instance.
+* `allow_relay`: Permits remote instances to subscribe to all public posts of your instance. This may increase the visibility of your instance.
 * `public`: Makes the client API in authenticated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. Note that there is a dependent setting restricting or allowing unauthenticated access to specific resources, see `restrict_unauthenticated` for more details.
 * `quarantined_instances`: List of ActivityPub instances where private (DMs, followers-only) activities will not be send.
 * `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML).
@ -65,6 +65,13 @@ To add configuration to your config file, you can copy it from the base config.
 * `show_reactions`: Let favourites and emoji reactions be viewed through the API (default: `true`).
 * `password_reset_token_validity`: The time after which reset tokens aren't accepted anymore, in seconds (default: one day).

+## :database
+* `improved_hashtag_timeline`: Setting to force toggle / force disable improved hashtags timeline. `:enabled` forces hashtags to be fetched from `hashtags` table for hashtags timeline. `:disabled` forces object-embedded hashtags to be used (slower). Keep it `:auto` for automatic behaviour (it is auto-set to `:enabled` [unless overridden] when HashtagsTableMigrator completes).
+
+## Background migrations
+* `populate_hashtags_table/sleep_interval_ms`: Sleep interval between each chunk of processed records in order to decrease the load on the system (defaults to 0 and should be keep default on most instances).
+* `populate_hashtags_table/fault_rate_allowance`: Max rate of failed objects to actually processed objects in order to enable the feature (any value from 0.0 which tolerates no errors to 1.0 which will enable the feature even if hashtags transfer failed for all records).
+
 ## Welcome
 * `direct_message`: - welcome message sent as a direct message.
  * `enabled`: Enables the send a direct message to a newly registered user. Defaults to `false`.
@ -117,6 +124,7 @@ To add configuration to your config file, you can copy it from the base config.
    * `Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy`: Rejects or delists posts based on their age when received. (See [`:mrf_object_age`](#mrf_object_age)).
    * `Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy`: Sets a default expiration on all posts made by users of the local instance. Requires `Pleroma.Workers.PurgeExpiredActivity` to be enabled for processing the scheduled delections.
    * `Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy`: Makes all bot posts to disappear from public timelines.
+    * `Pleroma.Web.ActivityPub.MRF.FollowBotPolicy`: Automatically follows newly discovered users from the specified bot account. Local accounts, locked accounts, and users with "#nobot" in their bio are respected and excluded from being followed.
 * `transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo).
 * `transparency_exclusions`: Exclude specific instance names from MRF transparency.  The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.

@ -203,6 +211,21 @@ config :pleroma, :mrf_user_allowlist, %{

 * `days`: Default global expiration time for all local Create activities (in days)

+#### :mrf_hashtag
+
+* `sensitive`: List of hashtags to mark activities as sensitive (default: `nsfw`)
+* `federated_timeline_removal`: List of hashtags to remove activities from the federated timeline (aka TWNK)
+* `reject`: List of hashtags to reject activities from
+
+Notes:
+- The hashtags in the configuration do not have a leading `#`.
+- This MRF Policy is always enabled, if you want to disable it you have to set empty lists
+
+#### :mrf_follow_bot
+
+* `follower_nickname`: The name of the bot account to use for following newly discovered users. Using `followbot` or similar is strongly suggested.
+
+
 ### :activitypub
 * `unfollow_blocked`: Whether blocks result in people getting unfollowed
 * `outgoing_blocks`: Whether to federate blocks to other instances
--- a/docs/configuration/mrf.md
+++ b/docs/configuration/mrf.md
@ -82,7 +82,7 @@ For example, here is a sample policy module which rewrites all messages to "new
 ```elixir
 defmodule Pleroma.Web.ActivityPub.MRF.RewritePolicy do
  @moduledoc "MRF policy which rewrites all Notes to have 'new message content'."
-  @behaviour Pleroma.Web.ActivityPub.MRF
+  @behaviour Pleroma.Web.ActivityPub.MRF.Policy

  # Catch messages which contain Note objects with actual data to filter.
  # Capture the object as `object`, the message content as `content` and the
--- a/docs/development/API/differences_in_mastoapi_responses.md
+++ b/docs/development/API/differences_in_mastoapi_responses.md
@ -38,6 +38,7 @@ Has these additional fields under the `pleroma` object:
 - `thread_muted`: true if the thread the post belongs to is muted
 - `emoji_reactions`: A list with emoji / reaction maps. The format is `{name: "☕", count: 1, me: true}`. Contains no information about the reacting users, for that use the `/statuses/:id/reactions` endpoint.
 - `parent_visible`: If the parent of this post is visible to the user or not.
+- `pinned_at`: a datetime (iso8601) when status was pinned, `null` otherwise.

 ## Scheduled statuses

@ -255,9 +256,29 @@ This information is returned in the `/api/v1/accounts/verify_credentials` endpoi

 *Pleroma supports refreshing tokens.*

-`POST /oauth/token`
+### POST `/oauth/token`

-Post here request with `grant_type=refresh_token` to obtain new access token. Returns an access token.
+You can obtain access tokens for a user in a few additional ways.
+
+#### Refreshing a token
+
+To obtain a new access token from a refresh token, pass `grant_type=refresh_token` with the following extra parameters:
+
+- `refresh_token`: The refresh token.
+
+#### Getting a token with a password
+
+To obtain a token from a user's password, pass `grant_type=password` with the following extra parameters:
+
+- `username`: Username to authenticate.
+- `password`: The user's password.
+
+#### Response body
+
+Additional fields are returned in the response:
+
+- `id`: The primary key of this token in Pleroma's database.
+- `me` (user tokens only): The ActivityPub ID of the user who owns the token.

 ## Account Registration

--- a/docs/development/API/pleroma_api.md
+++ b/docs/development/API/pleroma_api.md
@ -300,7 +300,7 @@ See [Admin-API](admin_api.md)
 * Note: Behaves exactly the same as `POST /api/v1/upload`.
  Can only accept images - any attempt to upload non-image files will be met with `HTTP 415 Unsupported Media Type`.

-## `/api/v1/pleroma/notification_settings`
+## `/api/pleroma/notification_settings`
 ### Updates user notification settings
 * Method `PUT`
 * Authentication: required
--- a/docs/index.md
+++ b/docs/index.md
@ -20,7 +20,7 @@ The default front-end used by Pleroma is Pleroma-FE. You can find more informati

 ### Mastodon interface
 If the Pleroma interface isn't your thing, or you're just trying something new but you want to keep using the familiar Mastodon interface, we got that too!
-Just add a "/web" after your instance url (e.g. <https://pleroma.soycaf.com/web>) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC!
+Just add a "/web" after your instance url (e.g. <https://pleroma.soykaf.com/web>) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC!
 The Mastodon interface is from the Glitch-soc fork. For more information on the Mastodon interface you can check the [Mastodon](https://docs.joinmastodon.org/) and [Glitch-soc](https://glitch-soc.github.io/docs/) documentation.

 Remember, what you see is only the frontend part of Mastodon, the backend is still Pleroma.
--- a/docs/installation/alpine_linux_en.md
+++ b/docs/installation/alpine_linux_en.md
@ -117,7 +117,7 @@ cd /opt/pleroma
 sudo -Hu pleroma mix deps.get
 ```

-* Generate the configuration: `sudo -Hu pleroma mix pleroma.instance gen`
+* Generate the configuration: `sudo -Hu pleroma MIX_ENV=prod mix pleroma.instance gen`
  * Answer with `yes` if it asks you to install `rebar3`.
  * This may take some time, because parts of pleroma get compiled first.
  * After that it will ask you a few questions about your instance and generates a configuration file in `config/generated_config.exs`.
@ -240,4 +240,4 @@ sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress

 ## Questions

-Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.
+Questions about the installation or didn’t it work as it should be, ask in [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat) via Matrix or **#pleroma** on **libera.chat** via IRC.
--- a/docs/installation/arch_linux_en.md
+++ b/docs/installation/arch_linux_en.md
@ -92,7 +92,7 @@ cd /opt/pleroma
 sudo -Hu pleroma mix deps.get
 ```

-* Generate the configuration: `sudo -Hu pleroma mix pleroma.instance gen`
+* Generate the configuration: `sudo -Hu pleroma MIX_ENV=prod mix pleroma.instance gen`
  * Answer with `yes` if it asks you to install `rebar3`.
  * This may take some time, because parts of pleroma get compiled first.
  * After that it will ask you a few questions about your instance and generates a configuration file in `config/generated_config.exs`.
@ -215,4 +215,4 @@ sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress

 ## Questions

-Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.
+Questions about the installation or didn’t it work as it should be, ask in [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat) via Matrix or **#pleroma** on **libera.chat** via IRC.
--- a/docs/installation/debian_based_en.md
+++ b/docs/installation/debian_based_en.md
@ -90,7 +90,7 @@ cd /opt/pleroma
 sudo -Hu pleroma mix deps.get
 ```

-* Generate the configuration: `sudo -Hu pleroma mix pleroma.instance gen`
+* Generate the configuration: `sudo -Hu pleroma MIX_ENV=prod mix pleroma.instance gen`
  * Answer with `yes` if it asks you to install `rebar3`.
  * This may take some time, because parts of pleroma get compiled first.
  * After that it will ask you a few questions about your instance and generates a configuration file in `config/generated_config.exs`.
@ -202,4 +202,4 @@ sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress

 ## Questions

-Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.
+Questions about the installation or didn’t it work as it should be, ask in [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat) via Matrix or **#pleroma** on **libera.chat** via IRC.
--- a/docs/installation/debian_based_jp.md
+++ b/docs/installation/debian_based_jp.md
@ -89,7 +89,7 @@ sudo -Hu pleroma mix deps.get

 * コンフィギュレーションを生成します。
 ```
-sudo -Hu pleroma mix pleroma.instance gen
+sudo -Hu pleroma MIX_ENV=prod mix pleroma.instance gen
 ```
    * rebar3をインストールしてもよいか聞かれたら、yesを入力してください。
    * このときにpleromaの一部がコンパイルされるため、この処理には時間がかかります。
@ -103,7 +103,7 @@ sudo -Hu pleroma mv config/{generated_config.exs,prod.secret.exs}

 * 先程のコマンドで、すでに `config/setup_db.psql` というファイルが作られています。このファイルをもとに、データベースを作成します。
 ```
-sudo -Hu pleroma mix pleroma.instance gen
+sudo -Hu pleroma MIX_ENV=prod mix pleroma.instance gen
 ```

 * そして、データベースのマイグレーションを実行します。
@ -191,5 +191,5 @@ sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress

 インストールについて質問がある、もしくは、うまくいかないときは、以下のところで質問できます。

-* [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org)
-* **Freenode** の **#pleroma** IRCチャンネル
+* [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat)
+* **libera.chat** の **#pleroma** IRCチャンネル
--- a/docs/installation/freebsd_en.md
+++ b/docs/installation/freebsd_en.md
@ -54,7 +54,7 @@ Configure Pleroma. Note that you need a domain name at this point:
 ```
 $ cd /home/pleroma/pleroma
 $ mix deps.get # Enter "y" when asked to install Hex
-$ mix pleroma.instance gen # You will be asked a few questions here.
+$ MIX_ENV=prod mix pleroma.instance gen # You will be asked a few questions here.
 $ cp config/generated_config.exs config/prod.secret.exs
 ```

@ -213,4 +213,4 @@ incorrect timestamps. You should have ntpd running.

 ## Questions

-Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.
+Questions about the installation or didn’t it work as it should be, ask in [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat) via Matrix or **#pleroma** on **libera.chat** via IRC.
--- a/docs/installation/gentoo_en.md
+++ b/docs/installation/gentoo_en.md
@ -135,7 +135,7 @@ pleroma$ mix deps.get
 * Generate the configuration:

 ```shell
-pleroma$ mix pleroma.instance gen
+pleroma$ MIX_ENV=prod mix pleroma.instance gen
 ```

  * Answer with `yes` if it asks you to install `rebar3`.
@ -298,4 +298,4 @@ If you opted to allow sudo for the `pleroma` user but would like to remove the a

 ## Questions

-Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.
+Questions about the installation or didn’t it work as it should be, ask in [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat) via Matrix or **#pleroma** on **libera.chat** via IRC.
--- a/docs/installation/netbsd_en.md
+++ b/docs/installation/netbsd_en.md
@ -71,7 +71,7 @@ Configure Pleroma. Note that you need a domain name at this point:
 ```
 $ cd /home/pleroma/pleroma
 $ mix deps.get
-$ mix pleroma.instance gen # You will be asked a few questions here.
+$ MIX_ENV=prod mix pleroma.instance gen # You will be asked a few questions here.
 ```

 Since Postgres is configured, we can now initialize the database. There should
@ -193,8 +193,6 @@ Run `# /etc/rc.d/pleroma start` to start Pleroma.

 Restart nginx with `# /etc/rc.d/nginx restart` and you should be up and running.

-If you need further help, contact niaa on freenode.
-
 Make sure your time is in sync, or other instances will receive your posts with
 incorrect timestamps. You should have ntpd running.

@ -208,4 +206,4 @@ incorrect timestamps. You should have ntpd running.

 ## Questions

-Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.
+Questions about the installation or didn’t it work as it should be, ask in [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat) via Matrix or **#pleroma** on **libera.chat** via IRC.
--- a/docs/installation/openbsd_en.md
+++ b/docs/installation/openbsd_en.md
@ -239,7 +239,7 @@ Enter a shell as \_pleroma (as root `su _pleroma -`) and enter pleroma's install
 Then follow the main installation guide:

  * run `mix deps.get`
-  * run `mix pleroma.instance gen` and enter your instance's information when asked
+  * run `MIX_ENV=prod mix pleroma.instance gen` and enter your instance's information when asked
  * copy config/generated\_config.exs to config/prod.secret.exs. The default values should be sufficient but you should edit it and check that everything seems OK.
  * exit your current shell back to a root one and run `psql -U postgres -f /home/_pleroma/pleroma/config/setup_db.psql` to setup the database.
  * return to a \_pleroma shell into pleroma's installation directory (`su _pleroma -;cd ~/pleroma`) and run `MIX_ENV=prod mix ecto.migrate`
@ -264,4 +264,4 @@ LC_ALL=en_US.UTF-8 MIX_ENV=prod mix pleroma.user new <username> <your@emailaddre

 ## Questions

-Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.
+Questions about the installation or didn’t it work as it should be, ask in [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat) via Matrix or **#pleroma** on **libera.chat** via IRC.
--- a/docs/installation/openbsd_fi.md
+++ b/docs/installation/openbsd_fi.md
@ -10,8 +10,8 @@ suositeltavaa tehdä komennon `doas` avulla, katso `doas (1)` ja `doas.conf (5)`
 Tästä eteenpäin oletuksena on, että domain "esimerkki.com" osoittaa
 serverin IP-osoitteeseen.

-Jos asennuksen kanssa on ongelmia, IRC-kanava #pleroma Freenodessa tai
-Matrix-kanava #freenode_#pleroma:matrix.org ovat hyviä paikkoja löytää apua
+Jos asennuksen kanssa on ongelmia, IRC-kanava #pleroma Libera.chat tai
+Matrix-kanava #pleroma:libera.chat ovat hyviä paikkoja löytää apua
 (englanniksi), `/msg eal kukkuu` jos haluat välttämättä puhua härmää.

 Asenna tarvittava ohjelmisto:
--- a/docs/installation/otp_en.md
+++ b/docs/installation/otp_en.md
@ -232,7 +232,7 @@ At this point if you open your (sub)domain in a browser you should see a 502 err

 If everything worked, you should see Pleroma-FE when visiting your domain. If that didn't happen, try reviewing the installation steps, starting Pleroma in the foreground and seeing if there are any errrors.

-Still doesn't work? Feel free to contact us on [#pleroma on freenode](https://irc.pleroma.social) or via matrix at <https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org>, you can also [file an issue on our Gitlab](https://git.pleroma.social/pleroma/pleroma-support/issues/new)
+Questions about the installation or didn’t it work as it should be, ask in [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat) via Matrix or **#pleroma** on **libera.chat** via IRC, you can also [file an issue on our Gitlab](https://git.pleroma.social/pleroma/pleroma-support/issues/new).

 ## Post installation

@ -290,7 +290,7 @@ nginx -t

 ## Create your first user and set as admin
 ```sh
-cd /opt/pleroma/bin
+cd /opt/pleroma
 su pleroma -s $SHELL -lc "./bin/pleroma_ctl user new joeuser joeuser@sld.tld --admin"
 ```
 This will create an account withe the username of 'joeuser' with the email address of joeuser@sld.tld, and set that user's account as an admin. This will result in a link that you can paste into the browser, which logs you in and enables you to set the password.
@ -301,4 +301,4 @@ This will create an account withe the username of 'joeuser' with the email addre

 ## Questions

-Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.
+Questions about the installation or didn’t it work as it should be, ask in [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat) via Matrix or **#pleroma** on **libera.chat** via IRC, you can also [file an issue on our Gitlab](https://git.pleroma.social/pleroma/pleroma-support/issues/new).
--- a/lib/mix/tasks/pleroma/config.ex
+++ b/lib/mix/tasks/pleroma/config.ex
@ -27,7 +27,7 @@ defmodule Mix.Tasks.Pleroma.Config do

      {opts, _} =
        OptionParser.parse!(options,
-          strict: [env: :string, delete: :boolean],
+          strict: [env: :string, delete: :boolean, path: :string],
          aliases: [d: :delete]
        )

@ -259,18 +259,43 @@ defmodule Mix.Tasks.Pleroma.Config do
  defp migrate_from_db(opts) do
    env = opts[:env] || Pleroma.Config.get(:env)

+    filename = "#{env}.exported_from_db.secret.exs"
+
    config_path =
-      if Pleroma.Config.get(:release) do
+      cond do
+        opts[:path] ->
+          opts[:path]
+
+        Pleroma.Config.get(:release) ->
          :config_path
          |> Pleroma.Config.get()
          |> Path.dirname()
-      else
+
+        true ->
          "config"
      end
-      |> Path.join("#{env}.exported_from_db.secret.exs")
+      |> Path.join(filename)

-    file = File.open!(config_path, [:write, :utf8])
+    with {:ok, file} <- File.open(config_path, [:write, :utf8]) do
+      write_config(file, config_path, opts)
+      shell_info("Database configuration settings have been exported to #{config_path}")
+    else
+      _ ->
+        shell_error("Impossible to save settings to this directory #{Path.dirname(config_path)}")
+        tmp_config_path = Path.join(System.tmp_dir!(), filename)
+        file = File.open!(tmp_config_path)

+        shell_info(
+          "Saving database configuration settings to #{tmp_config_path}. Copy it to the #{
+            Path.dirname(config_path)
+          } manually."
+        )
+
+        write_config(file, tmp_config_path, opts)
+    end
+  end
+
+  defp write_config(file, path, opts) do
    IO.write(file, config_header())

    ConfigDB
@ -278,11 +303,7 @@ defmodule Mix.Tasks.Pleroma.Config do
    |> Enum.each(&write_and_delete(&1, file, opts[:delete]))

    :ok = File.close(file)
-    System.cmd("mix", ["format", config_path])
-
-    shell_info(
-      "Database configuration settings have been exported to config/#{env}.exported_from_db.secret.exs"
-    )
+    System.cmd("mix", ["format", path])
  end

  if Code.ensure_loaded?(Config.Reader) do
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@ -8,10 +8,13 @@ defmodule Mix.Tasks.Pleroma.Database do
  alias Pleroma.Object
  alias Pleroma.Repo
  alias Pleroma.User
+
  require Logger
  require Pleroma.Constants
+
  import Ecto.Query
  import Mix.Pleroma
+
  use Mix.Task

  @shortdoc "A collection of database related tasks"
@ -93,6 +96,15 @@ defmodule Mix.Tasks.Pleroma.Database do
    )
    |> Repo.delete_all(timeout: :infinity)

+    prune_hashtags_query = """
+    DELETE FROM hashtags AS ht
+    WHERE NOT EXISTS (
+      SELECT 1 FROM hashtags_objects hto
+      WHERE ht.id = hto.hashtag_id)
+    """
+
+    Repo.query(prune_hashtags_query)
+
    if Keyword.get(options, :vacuum) do
      Maintenance.vacuum("full")
    end
@ -214,4 +226,32 @@ defmodule Mix.Tasks.Pleroma.Database do
      shell_info('Done.')
    end
  end
+
+  # Rolls back a specific migration (leaving subsequent migrations applied).
+  # WARNING: imposes a risk of unrecoverable data loss — proceed at your own responsibility.
+  # Based on https://stackoverflow.com/a/53825840
+  def run(["rollback", version]) do
+    prompt = "SEVERE WARNING: this operation may result in unrecoverable data loss. Continue?"
+
+    if shell_prompt(prompt, "n") in ~w(Yn Y y) do
+      {_, result, _} =
+        Ecto.Migrator.with_repo(Pleroma.Repo, fn repo ->
+          version = String.to_integer(version)
+          re = ~r/^#{version}_.*\.exs/
+          path = Ecto.Migrator.migrations_path(repo)
+
+          with {_, "" <> file} <- {:find, Enum.find(File.ls!(path), &String.match?(&1, re))},
+               {_, [{mod, _} | _]} <- {:compile, Code.compile_file(Path.join(path, file))},
+               {_, :ok} <- {:rollback, Ecto.Migrator.down(repo, version, mod)} do
+            {:ok, "Reversed migration: #{file}"}
+          else
+            {:find, _} -> {:error, "No migration found with version prefix: #{version}"}
+            {:compile, e} -> {:error, "Problem compiling migration module: #{inspect(e)}"}
+            {:rollback, e} -> {:error, "Problem reversing migration: #{inspect(e)}"}
+          end
+        end)
+
+      shell_info(inspect(result))
+    end
+  end
 end
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@ -113,6 +113,7 @@ defmodule Pleroma.Activity do
    from([a] in query,
      left_join: b in Bookmark,
      on: b.user_id == ^user.id and b.activity_id == a.id,
+      as: :bookmark,
      preload: [bookmark: b]
    )
  end
@ -123,6 +124,7 @@ defmodule Pleroma.Activity do
    from([a] in query,
      left_join: r in ReportNote,
      on: a.id == r.activity_id,
+      as: :report_note,
      preload: [report_notes: r]
    )
  end
@ -182,38 +184,46 @@ defmodule Pleroma.Activity do
    |> Repo.one()
  end

-  @spec get_by_id(String.t()) :: Activity.t() | nil
-  def get_by_id(id) do
-    case FlakeId.flake_id?(id) do
-      true ->
-        Activity
-        |> where([a], a.id == ^id)
-        |> restrict_deactivated_users()
-        |> Repo.one()
+  @doc """
+  Gets activity by ID, doesn't load activities from deactivated actors by default.
+  """
+  @spec get_by_id(String.t(), keyword()) :: t() | nil
+  def get_by_id(id, opts \\ [filter: [:restrict_deactivated]]), do: get_by_id_with_opts(id, opts)

-      _ ->
-        nil
-    end
+  @spec get_by_id_with_user_actor(String.t()) :: t() | nil
+  def get_by_id_with_user_actor(id), do: get_by_id_with_opts(id, preload: [:user_actor])
+
+  @spec get_by_id_with_object(String.t()) :: t() | nil
+  def get_by_id_with_object(id), do: get_by_id_with_opts(id, preload: [:object])
+
+  defp get_by_id_with_opts(id, opts) do
+    if FlakeId.flake_id?(id) do
+      query = Queries.by_id(id)
+
+      with_filters_query =
+        if is_list(opts[:filter]) do
+          Enum.reduce(opts[:filter], query, fn
+            {:type, type}, acc -> Queries.by_type(acc, type)
+            :restrict_deactivated, acc -> restrict_deactivated_users(acc)
+            _, acc -> acc
+          end)
+        else
+          query
        end

-  def get_by_id_with_user_actor(id) do
-    case FlakeId.flake_id?(id) do
-      true ->
-        Activity
-        |> where([a], a.id == ^id)
-        |> with_preloaded_user_actor()
-        |> Repo.one()
-
-      _ ->
-        nil
-    end
+      with_preloads_query =
+        if is_list(opts[:preload]) do
+          Enum.reduce(opts[:preload], with_filters_query, fn
+            :user_actor, acc -> with_preloaded_user_actor(acc)
+            :object, acc -> with_preloaded_object(acc)
+            _, acc -> acc
+          end)
+        else
+          with_filters_query
        end

-  def get_by_id_with_object(id) do
-    Activity
-    |> where(id: ^id)
-    |> with_preloaded_object()
-    |> Repo.one()
+      Repo.one(with_preloads_query)
+    end
  end

  def all_by_ids_with_object(ids) do
@ -267,6 +277,11 @@ defmodule Pleroma.Activity do

  def get_create_by_object_ap_id_with_object(_), do: nil

+  @spec create_by_id_with_object(String.t()) :: t() | nil
+  def create_by_id_with_object(id) do
+    get_by_id_with_opts(id, preload: [:object], filter: [type: "Create"])
+  end
+
  defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}) do
    get_create_by_object_ap_id_with_object(ap_id)
  end
@ -366,12 +381,6 @@ defmodule Pleroma.Activity do
    end
  end

-  @spec pinned_by_actor?(Activity.t()) :: boolean()
-  def pinned_by_actor?(%Activity{} = activity) do
-    actor = user_actor(activity)
-    activity.id in actor.pinned_activities
-  end
-
  @spec get_by_object_ap_id_with_object(String.t()) :: t() | nil
  def get_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do
    ap_id
@ -382,4 +391,13 @@ defmodule Pleroma.Activity do
  end

  def get_by_object_ap_id_with_object(_), do: nil
+
+  @spec add_by_params_query(String.t(), String.t(), String.t()) :: Ecto.Query.t()
+  def add_by_params_query(object_id, actor, target) do
+    object_id
+    |> Queries.by_object_id()
+    |> Queries.by_type("Add")
+    |> Queries.by_actor(actor)
+    |> where([a], fragment("?->>'target' = ?", a.data, ^target))
+  end
 end
--- a/lib/pleroma/activity/html.ex
+++ b/lib/pleroma/activity/html.ex
@ -0,0 +1,45 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Activity.HTML do
+  alias Pleroma.HTML
+  alias Pleroma.Object
+
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
+  def get_cached_scrubbed_html_for_activity(
+        content,
+        scrubbers,
+        activity,
+        key \\ "",
+        callback \\ fn x -> x end
+      ) do
+    key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}"
+
+    @cachex.fetch!(:scrubber_cache, key, fn _key ->
+      object = Object.normalize(activity, fetch: false)
+      HTML.ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback)
+    end)
+  end
+
+  def get_cached_stripped_html_for_activity(content, activity, key) do
+    get_cached_scrubbed_html_for_activity(
+      content,
+      FastSanitize.Sanitizer.StripTags,
+      activity,
+      key,
+      &HtmlEntities.decode/1
+    )
+  end
+
+  defp generate_scrubber_signature(scrubber) when is_atom(scrubber) do
+    generate_scrubber_signature([scrubber])
+  end
+
+  defp generate_scrubber_signature(scrubbers) do
+    Enum.reduce(scrubbers, "", fn scrubber, signature ->
+      "#{signature}#{to_string(scrubber)}"
+    end)
+  end
+end
--- a/lib/pleroma/activity/ir/topics.ex
+++ b/lib/pleroma/activity/ir/topics.ex
@ -48,14 +48,12 @@ defmodule Pleroma.Activity.Ir.Topics do
    tags
  end

-  defp hashtags_to_topics(%{data: %{"tag" => tags}}) do
-    tags
-    |> Enum.filter(&is_bitstring(&1))
-    |> Enum.map(fn tag -> "hashtag:" <> tag end)
+  defp hashtags_to_topics(object) do
+    object
+    |> Object.hashtags()
+    |> Enum.map(fn hashtag -> "hashtag:" <> hashtag end)
  end

-  defp hashtags_to_topics(_), do: []
-
  defp remote_topics(%{local: true}), do: []

  defp remote_topics(%{actor: actor}) when is_binary(actor),
--- a/lib/pleroma/activity/queries.ex
+++ b/lib/pleroma/activity/queries.ex
@ -14,6 +14,11 @@ defmodule Pleroma.Activity.Queries do
  alias Pleroma.Activity
  alias Pleroma.User

+  @spec by_id(query(), String.t()) :: query()
+  def by_id(query \\ Activity, id) do
+    from(a in query, where: a.id == ^id)
+  end
+
  @spec by_ap_id(query, String.t()) :: query
  def by_ap_id(query \\ Activity, ap_id) do
    from(
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@ -25,7 +25,7 @@ defmodule Pleroma.Application do
    if Process.whereis(Pleroma.Web.Endpoint) do
      case Config.get([:http, :user_agent], :default) do
        :default ->
-          info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>"
+          info = "#{Pleroma.Web.Endpoint.url()} <#{Config.get([:instance, :email], "")}>"
          named_version() <> "; " <> info

        custom ->
@ -102,10 +102,8 @@ defmodule Pleroma.Application do
        ] ++
        task_children(@mix_env) ++
        dont_run_in_test(@mix_env) ++
-        chat_child(chat_enabled?()) ++
-        [
-          Pleroma.Gopher.Server
-        ]
+        shout_child(shout_enabled?()) ++
+        [Pleroma.Gopher.Server]

    # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
    # for other strategies and supported options
@ -218,7 +216,7 @@ defmodule Pleroma.Application do
      type: :worker
    }

-  defp chat_enabled?, do: Config.get([:chat, :enabled])
+  defp shout_enabled?, do: Config.get([:shout, :enabled])

  defp dont_run_in_test(env) when env in [:test, :benchmark], do: []

@ -230,17 +228,23 @@ defmodule Pleroma.Application do
         keys: :duplicate,
         partitions: System.schedulers_online()
       ]}
+    ] ++ background_migrators()
+  end
+
+  defp background_migrators do
+    [
+      Pleroma.Migrators.HashtagsTableMigrator
    ]
  end

-  defp chat_child(true) do
+  defp shout_child(true) do
    [
-      Pleroma.Web.ChatChannel.ChatChannelState,
+      Pleroma.Web.ShoutChannel.ShoutChannelState,
      {Phoenix.PubSub, [name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2]}
    ]
  end

-  defp chat_child(_), do: []
+  defp shout_child(_), do: []

  defp task_children(:test) do
    [
--- a/lib/pleroma/application_requirements.ex
+++ b/lib/pleroma/application_requirements.ex
@ -34,15 +34,16 @@ defmodule Pleroma.ApplicationRequirements do
  defp check_welcome_message_config!(:ok) do
    if Pleroma.Config.get([:welcome, :email, :enabled], false) and
         not Pleroma.Emails.Mailer.enabled?() do
-      Logger.error("""
-      To send welcome email do you need to enable mail.
-      \nconfig :pleroma, Pleroma.Emails.Mailer, enabled: true
-      """)
+      Logger.warn("""
+      To send welcome emails, you need to enable the mailer.
+      Welcome emails will NOT be sent with the current config.

-      {:error, "The mail disabled."}
-    else
-      :ok
+      Enable the mailer:
+        config :pleroma, Pleroma.Emails.Mailer, enabled: true
+      """)
    end
+
+    :ok
  end

  defp check_welcome_message_config!(result), do: result
@ -51,18 +52,21 @@ defmodule Pleroma.ApplicationRequirements do
  #
  def check_confirmation_accounts!(:ok) do
    if Pleroma.Config.get([:instance, :account_activation_required]) &&
-         not Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do
-      Logger.error(
-        "Account activation enabled, but no Mailer settings enabled.\n" <>
-          "Please set config :pleroma, :instance, account_activation_required: false\n" <>
-          "Otherwise setup and enable Mailer."
-      )
+         not Pleroma.Emails.Mailer.enabled?() do
+      Logger.warn("""
+      Account activation is required, but the mailer is disabled.
+      Users will NOT be able to confirm their accounts with this config.
+      Either disable account activation or enable the mailer.

-      {:error,
-       "Account activation enabled, but Mailer is disabled. Cannot send confirmation emails."}
-    else
-      :ok
+      Disable account activation:
+        config :pleroma, :instance, account_activation_required: false
+
+      Enable the mailer:
+        config :pleroma, Pleroma.Emails.Mailer, enabled: true
+      """)
    end
+
+    :ok
  end

  def check_confirmation_accounts!(result), do: result
@ -160,9 +164,12 @@ defmodule Pleroma.ApplicationRequirements do

  defp check_system_commands!(:ok) do
    filter_commands_statuses = [
-      check_filter(Pleroma.Upload.Filters.Exiftool, "exiftool"),
-      check_filter(Pleroma.Upload.Filters.Mogrify, "mogrify"),
-      check_filter(Pleroma.Upload.Filters.Mogrifun, "mogrify")
+      check_filter(Pleroma.Upload.Filter.Exiftool, "exiftool"),
+      check_filter(Pleroma.Upload.Filter.Mogrify, "mogrify"),
+      check_filter(Pleroma.Upload.Filter.Mogrifun, "mogrify"),
+      check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "mogrify"),
+      check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "convert"),
+      check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "ffprobe")
    ]

    preview_proxy_commands_status =
--- a/lib/pleroma/config.ex
+++ b/lib/pleroma/config.ex
@ -99,4 +99,8 @@ defmodule Pleroma.Config do
  def oauth_consumer_strategies, do: get([:auth, :oauth_consumer_strategies], [])

  def oauth_consumer_enabled?, do: oauth_consumer_strategies() != []
+
+  def feature_enabled?(feature_name) do
+    get([:features, feature_name]) not in [nil, false, :disabled, :auto]
+  end
 end
--- a/lib/pleroma/config/deprecation_warnings.ex
+++ b/lib/pleroma/config/deprecation_warnings.ex
@ -41,7 +41,8 @@ defmodule Pleroma.Config.DeprecationWarnings do
         :ok <- check_gun_pool_options(),
         :ok <- check_activity_expiration_config(),
         :ok <- check_remote_ip_plug_name(),
-         :ok <- check_uploders_s3_public_endpoint() do
+         :ok <- check_uploders_s3_public_endpoint(),
+         :ok <- check_old_chat_shoutbox() do
      :ok
    else
      _ ->
@ -215,4 +216,27 @@ defmodule Pleroma.Config.DeprecationWarnings do
      :ok
    end
  end
+
+  @spec check_old_chat_shoutbox() :: :ok | nil
+  def check_old_chat_shoutbox do
+    instance_config = Pleroma.Config.get([:instance])
+    chat_config = Pleroma.Config.get([:chat]) || []
+
+    use_old_config =
+      Keyword.has_key?(instance_config, :chat_limit) or
+        Keyword.has_key?(chat_config, :enabled)
+
+    if use_old_config do
+      Logger.error("""
+      !!!DEPRECATION WARNING!!!
+      Your config is using the old namespace for the Shoutbox configuration. You need to convert to the new namespace. e.g.,
+      \n* `config :pleroma, :chat, enabled` and `config :pleroma, :instance, chat_limit` are now equal to:
+      \n* `config :pleroma, :shout, enabled` and `config :pleroma, :shout, limit`
+      """)
+
+      :error
+    else
+      :ok
+    end
+  end
 end
--- a/lib/pleroma/config/loader.ex
+++ b/lib/pleroma/config/loader.ex
@ -3,9 +3,11 @@
 # SPDX-License-Identifier: AGPL-3.0-only

 defmodule Pleroma.Config.Loader do
+  # These modules are only being used as keys here (for equality check),
+  # so it's okay to use `Module.concat/1` to have the compiler ignore them.
  @reject_keys [
-    Pleroma.Repo,
-    Pleroma.Web.Endpoint,
+    Module.concat(["Pleroma.Repo"]),
+    Module.concat(["Pleroma.Web.Endpoint"]),
    :env,
    :configurable_from_database,
    :database,
--- a/lib/pleroma/config/release_runtime_provider.ex
+++ b/lib/pleroma/config/release_runtime_provider.ex
@ -1,6 +1,6 @@
 defmodule Pleroma.Config.ReleaseRuntimeProvider do
  @moduledoc """
-  Imports `runtime.exs` and `{env}.exported_from_db.secret.exs` for elixir releases.
+  Imports runtime config and `{env}.exported_from_db.secret.exs` for releases.
  """
  @behaviour Config.Provider

@ -8,10 +8,11 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do
  def init(opts), do: opts

  @impl true
-  def load(config, _opts) do
+  def load(config, opts) do
    with_defaults = Config.Reader.merge(config, Pleroma.Config.Holder.release_defaults())

-    config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs"
+    config_path =
+      opts[:config_path] || System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs"

    with_runtime_config =
      if File.exists?(config_path) do
@ -24,7 +25,7 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do
        warning = [
          IO.ANSI.red(),
          IO.ANSI.bright(),
-          "!!! #{config_path} not found! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file",
+          "!!! Config path is not declared! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file",
          IO.ANSI.reset()
        ]

@ -33,13 +34,14 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do
      end

    exported_config_path =
+      opts[:exported_config_path] ||
        config_path
        |> Path.dirname()
-      |> Path.join("prod.exported_from_db.secret.exs")
+        |> Path.join("#{Pleroma.Config.get(:env)}.exported_from_db.secret.exs")

    with_exported =
      if File.exists?(exported_config_path) do
-        exported_config = Config.Reader.read!(with_runtime_config)
+        exported_config = Config.Reader.read!(exported_config_path)
        Config.Reader.merge(with_runtime_config, exported_config)
      else
        with_runtime_config
--- a/lib/pleroma/config/transfer_task.ex
+++ b/lib/pleroma/config/transfer_task.ex
@ -13,9 +13,10 @@ defmodule Pleroma.Config.TransferTask do

  @type env() :: :test | :benchmark | :dev | :prod

-  @reboot_time_keys [
+  defp reboot_time_keys,
+    do: [
      {:pleroma, :hackney_pools},
-    {:pleroma, :chat},
+      {:pleroma, :shout},
      {:pleroma, Oban},
      {:pleroma, :rate_limit},
      {:pleroma, :markup},
@ -24,7 +25,8 @@ defmodule Pleroma.Config.TransferTask do
      {:pleroma, :connections_pool}
    ]

-  @reboot_time_subkeys [
+  defp reboot_time_subkeys,
+    do: [
      {:pleroma, Pleroma.Captcha, [:seconds_valid]},
      {:pleroma, Pleroma.Upload, [:proxy_remote]},
      {:pleroma, :instance, [:upload_limit]},
@ -165,12 +167,12 @@ defmodule Pleroma.Config.TransferTask do
  end

  defp group_and_key_need_reboot?(group, key) do
-    Enum.any?(@reboot_time_keys, fn {g, k} -> g == group and k == key end)
+    Enum.any?(reboot_time_keys(), fn {g, k} -> g == group and k == key end)
  end

  defp group_and_subkey_need_reboot?(group, key, value) do
    Keyword.keyword?(value) and
-      Enum.any?(@reboot_time_subkeys, fn {g, k, subkeys} ->
+      Enum.any?(reboot_time_subkeys(), fn {g, k, subkeys} ->
        g == group and k == key and
          Enum.any?(Keyword.keys(value), &(&1 in subkeys))
      end)
--- a/lib/pleroma/config_db.ex
+++ b/lib/pleroma/config_db.ex
@ -387,6 +387,6 @@ defmodule Pleroma.ConfigDB do
  @spec module_name?(String.t()) :: boolean()
  def module_name?(string) do
    Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Quack|Ueberauth|Swoosh)\./, string) or
-      string in ["Oban", "Ueberauth", "ExSyslogger"]
+      string in ["Oban", "Ueberauth", "ExSyslogger", "ConcurrentLimiter"]
  end
 end
--- a/lib/pleroma/constants.ex
+++ b/lib/pleroma/constants.ex
@ -27,6 +27,4 @@ defmodule Pleroma.Constants do
    do:
      ~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc embed.js embed.css)
  )
-
-  def as_local_public, do: Pleroma.Web.base_url() <> "/#Public"
 end
--- a/lib/pleroma/data_migration.ex
+++ b/lib/pleroma/data_migration.ex
@ -0,0 +1,45 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.DataMigration do
+  use Ecto.Schema
+
+  alias Pleroma.DataMigration
+  alias Pleroma.DataMigration.State
+  alias Pleroma.Repo
+
+  import Ecto.Changeset
+  import Ecto.Query
+
+  schema "data_migrations" do
+    field(:name, :string)
+    field(:state, State, default: :pending)
+    field(:feature_lock, :boolean, default: false)
+    field(:params, :map, default: %{})
+    field(:data, :map, default: %{})
+
+    timestamps()
+  end
+
+  def changeset(data_migration, params \\ %{}) do
+    data_migration
+    |> cast(params, [:name, :state, :feature_lock, :params, :data])
+    |> validate_required([:name])
+    |> unique_constraint(:name)
+  end
+
+  def update_one_by_id(id, params \\ %{}) do
+    with {1, _} <-
+           from(dm in DataMigration, where: dm.id == ^id)
+           |> Repo.update_all(set: params) do
+      :ok
+    end
+  end
+
+  def get_by_name(name) do
+    Repo.get_by(DataMigration, name: name)
+  end
+
+  def populate_hashtags_table, do: get_by_name("populate_hashtags_table")
+end
--- a/lib/pleroma/delivery.ex
+++ b/lib/pleroma/delivery.ex
@ -9,7 +9,6 @@ defmodule Pleroma.Delivery do
  alias Pleroma.Object
  alias Pleroma.Repo
  alias Pleroma.User
-  alias Pleroma.User

  import Ecto.Changeset
  import Ecto.Query
--- a/lib/pleroma/earmark_renderer.ex
+++ b/lib/pleroma/earmark_renderer.ex
@ -1,256 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-#
-# This file is derived from Earmark, under the following copyright:
-# Copyright © 2014 Dave Thomas, The Pragmatic Programmers
-# SPDX-License-Identifier: Apache-2.0
-# Upstream: https://github.com/pragdave/earmark/blob/master/lib/earmark/html_renderer.ex
-defmodule Pleroma.EarmarkRenderer do
-  @moduledoc false
-
-  alias Earmark.Block
-  alias Earmark.Context
-  alias Earmark.HtmlRenderer
-  alias Earmark.Options
-
-  import Earmark.Inline, only: [convert: 3]
-  import Earmark.Helpers.HtmlHelpers
-  import Earmark.Message, only: [add_messages_from: 2, get_messages: 1, set_messages: 2]
-  import Earmark.Context, only: [append: 2, set_value: 2]
-  import Earmark.Options, only: [get_mapper: 1]
-
-  @doc false
-  def render(blocks, %Context{options: %Options{}} = context) do
-    messages = get_messages(context)
-
-    {contexts, html} =
-      get_mapper(context.options).(
-        blocks,
-        &render_block(&1, put_in(context.options.messages, []))
-      )
-      |> Enum.unzip()
-
-    all_messages =
-      contexts
-      |> Enum.reduce(messages, fn ctx, messages1 -> messages1 ++ get_messages(ctx) end)
-
-    {put_in(context.options.messages, all_messages), html |> IO.iodata_to_binary()}
-  end
-
-  #############
-  # Paragraph #
-  #############
-  defp render_block(%Block.Para{lnb: lnb, lines: lines, attrs: attrs}, context) do
-    lines = convert(lines, lnb, context)
-    add_attrs(lines, "<p>#{lines.value}</p>", attrs, [], lnb)
-  end
-
-  ########
-  # Html #
-  ########
-  defp render_block(%Block.Html{html: html}, context) do
-    {context, html}
-  end
-
-  defp render_block(%Block.HtmlComment{lines: lines}, context) do
-    {context, lines}
-  end
-
-  defp render_block(%Block.HtmlOneline{html: html}, context) do
-    {context, html}
-  end
-
-  #########
-  # Ruler #
-  #########
-  defp render_block(%Block.Ruler{lnb: lnb, attrs: attrs}, context) do
-    add_attrs(context, "<hr />", attrs, [], lnb)
-  end
-
-  ###########
-  # Heading #
-  ###########
-  defp render_block(
-         %Block.Heading{lnb: lnb, level: level, content: content, attrs: attrs},
-         context
-       ) do
-    converted = convert(content, lnb, context)
-    html = "<h#{level}>#{converted.value}</h#{level}>"
-    add_attrs(converted, html, attrs, [], lnb)
-  end
-
-  ##############
-  # Blockquote #
-  ##############
-
-  defp render_block(%Block.BlockQuote{lnb: lnb, blocks: blocks, attrs: attrs}, context) do
-    {context1, body} = render(blocks, context)
-    html = "<blockquote>#{body}</blockquote>"
-    add_attrs(context1, html, attrs, [], lnb)
-  end
-
-  #########
-  # Table #
-  #########
-
-  defp render_block(
-         %Block.Table{lnb: lnb, header: header, rows: rows, alignments: aligns, attrs: attrs},
-         context
-       ) do
-    {context1, html} = add_attrs(context, "<table>", attrs, [], lnb)
-    context2 = set_value(context1, html)
-
-    context3 =
-      if header do
-        append(add_trs(append(context2, "<thead>"), [header], "th", aligns, lnb), "</thead>")
-      else
-        # Maybe an error, needed append(context, html)
-        context2
-      end
-
-    context4 = append(add_trs(append(context3, "<tbody>"), rows, "td", aligns, lnb), "</tbody>")
-
-    {context4, [context4.value, "</table>"]}
-  end
-
-  ########
-  # Code #
-  ########
-
-  defp render_block(
-         %Block.Code{lnb: lnb, language: language, attrs: attrs} = block,
-         %Context{options: options} = context
-       ) do
-    class =
-      if language, do: ~s{ class="#{code_classes(language, options.code_class_prefix)}"}, else: ""
-
-    tag = ~s[<pre><code#{class}>]
-    lines = options.render_code.(block)
-    html = ~s[#{tag}#{lines}</code></pre>]
-    add_attrs(context, html, attrs, [], lnb)
-  end
-
-  #########
-  # Lists #
-  #########
-
-  defp render_block(
-         %Block.List{lnb: lnb, type: type, blocks: items, attrs: attrs, start: start},
-         context
-       ) do
-    {context1, content} = render(items, context)
-    html = "<#{type}#{start}>#{content}</#{type}>"
-    add_attrs(context1, html, attrs, [], lnb)
-  end
-
-  # format a single paragraph list item, and remove the para tags
-  defp render_block(
-         %Block.ListItem{lnb: lnb, blocks: blocks, spaced: false, attrs: attrs},
-         context
-       )
-       when length(blocks) == 1 do
-    {context1, content} = render(blocks, context)
-    content = Regex.replace(~r{</?p>}, content, "")
-    html = "<li>#{content}</li>"
-    add_attrs(context1, html, attrs, [], lnb)
-  end
-
-  # format a spaced list item
-  defp render_block(%Block.ListItem{lnb: lnb, blocks: blocks, attrs: attrs}, context) do
-    {context1, content} = render(blocks, context)
-    html = "<li>#{content}</li>"
-    add_attrs(context1, html, attrs, [], lnb)
-  end
-
-  ##################
-  # Footnote Block #
-  ##################
-
-  defp render_block(%Block.FnList{blocks: footnotes}, context) do
-    items =
-      Enum.map(footnotes, fn note ->
-        blocks = append_footnote_link(note)
-        %Block.ListItem{attrs: "#fn:#{note.number}", type: :ol, blocks: blocks}
-      end)
-
-    {context1, html} = render_block(%Block.List{type: :ol, blocks: items}, context)
-    {context1, Enum.join([~s[<div class="footnotes">], "<hr />", html, "</div>"])}
-  end
-
-  #######################################
-  # Isolated IALs are rendered as paras #
-  #######################################
-
-  defp render_block(%Block.Ial{verbatim: verbatim}, context) do
-    {context, "<p>{:#{verbatim}}</p>"}
-  end
-
-  ####################
-  # IDDef is ignored #
-  ####################
-
-  defp render_block(%Block.IdDef{}, context), do: {context, ""}
-
-  #####################################
-  # And here are the inline renderers #
-  #####################################
-
-  defdelegate br, to: HtmlRenderer
-  defdelegate codespan(text), to: HtmlRenderer
-  defdelegate em(text), to: HtmlRenderer
-  defdelegate strong(text), to: HtmlRenderer
-  defdelegate strikethrough(text), to: HtmlRenderer
-
-  defdelegate link(url, text), to: HtmlRenderer
-  defdelegate link(url, text, title), to: HtmlRenderer
-
-  defdelegate image(path, alt, title), to: HtmlRenderer
-
-  defdelegate footnote_link(ref, backref, number), to: HtmlRenderer
-
-  # Table rows
-  defp add_trs(context, rows, tag, aligns, lnb) do
-    numbered_rows =
-      rows
-      |> Enum.zip(Stream.iterate(lnb, &(&1 + 1)))
-
-    numbered_rows
-    |> Enum.reduce(context, fn {row, lnb}, ctx ->
-      append(add_tds(append(ctx, "<tr>"), row, tag, aligns, lnb), "</tr>")
-    end)
-  end
-
-  defp add_tds(context, row, tag, aligns, lnb) do
-    Enum.reduce(1..length(row), context, add_td_fn(row, tag, aligns, lnb))
-  end
-
-  defp add_td_fn(row, tag, aligns, lnb) do
-    fn n, ctx ->
-      style =
-        case Enum.at(aligns, n - 1, :default) do
-          :default -> ""
-          align -> " style=\"text-align: #{align}\""
-        end
-
-      col = Enum.at(row, n - 1)
-      converted = convert(col, lnb, set_messages(ctx, []))
-      append(add_messages_from(ctx, converted), "<#{tag}#{style}>#{converted.value}</#{tag}>")
-    end
-  end
-
-  ###############################
-  # Append Footnote Return Link #
-  ###############################
-
-  defdelegate append_footnote_link(note), to: HtmlRenderer
-  defdelegate append_footnote_link(note, fnlink), to: HtmlRenderer
-
-  defdelegate render_code(lines), to: HtmlRenderer
-
-  defp code_classes(language, prefix) do
-    ["" | String.split(prefix || "")]
-    |> Enum.map(fn pfx -> "#{pfx}#{language}" end)
-    |> Enum.join(" ")
-  end
-end
--- a/lib/pleroma/ecto_enums.ex
+++ b/lib/pleroma/ecto_enums.ex
@ -17,3 +17,11 @@ defenum(Pleroma.FollowingRelationship.State,
  follow_accept: 2,
  follow_reject: 3
 )
+
+defenum(Pleroma.DataMigration.State,
+  pending: 1,
+  running: 2,
+  complete: 3,
+  failed: 4,
+  manual: 5
+)
--- a/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex
+++ b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex
@ -13,21 +13,33 @@ defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Recipients do
    cast([object])
  end

+  def cast(object) when is_map(object) do
+    case ObjectID.cast(object) do
+      {:ok, data} -> {:ok, [data]}
+      _ -> :error
+    end
+  end
+
  def cast(data) when is_list(data) do
+    data =
      data
-    |> Enum.reduce_while({:ok, []}, fn element, {:ok, list} ->
+      |> Enum.reduce_while([], fn element, list ->
        case ObjectID.cast(element) do
          {:ok, id} ->
-          {:cont, {:ok, [id | list]}}
+            {:cont, [id | list]}

          _ ->
-          {:halt, :error}
+            {:cont, list}
        end
      end)
+      |> Enum.sort()
+      |> Enum.uniq()
+
+    {:ok, data}
  end

-  def cast(_) do
-    :error
+  def cast(data) do
+    {:error, data}
  end

  def dump(data) do
--- a/lib/pleroma/emails/admin_email.ex
+++ b/lib/pleroma/emails/admin_email.ex
@ -73,7 +73,7 @@ defmodule Pleroma.Emails.AdminEmail do
    #{comment_html}
    #{statuses_html}
    <p>
-    <a href="#{Pleroma.Web.base_url()}/pleroma/admin/#/reports/index">View Reports in AdminFE</a>
+    <a href="#{Pleroma.Web.Endpoint.url()}/pleroma/admin/#/reports/index">View Reports in AdminFE</a>
    """

    new()
@ -87,7 +87,7 @@ defmodule Pleroma.Emails.AdminEmail do
    html_body = """
    <p>New account for review: <a href="#{account.ap_id}">@#{account.nickname}</a></p>
    <blockquote>#{HTML.strip_tags(account.registration_reason)}</blockquote>
-    <a href="#{Pleroma.Web.base_url()}/pleroma/admin/#/users/#{account.id}/">Visit AdminFE</a>
+    <a href="#{Pleroma.Web.Endpoint.url()}/pleroma/admin/#/users/#{account.id}/">Visit AdminFE</a>
    """

    new()
--- a/lib/pleroma/emails/user_email.ex
+++ b/lib/pleroma/emails/user_email.ex
@ -5,15 +5,22 @@
 defmodule Pleroma.Emails.UserEmail do
  @moduledoc "User emails"

-  use Phoenix.Swoosh, view: Pleroma.Web.EmailView, layout: {Pleroma.Web.LayoutView, :email}
-
  alias Pleroma.Config
  alias Pleroma.User
  alias Pleroma.Web.Endpoint
  alias Pleroma.Web.Router

+  import Swoosh.Email
+  import Phoenix.Swoosh, except: [render_body: 3]
  import Pleroma.Config.Helpers, only: [instance_name: 0, sender: 0]

+  def render_body(email, template, assigns \\ %{}) do
+    email
+    |> put_new_layout({Pleroma.Web.LayoutView, :email})
+    |> put_new_view(Pleroma.Web.EmailView)
+    |> Phoenix.Swoosh.render_body(template, assigns)
+  end
+
  defp recipient(email, nil), do: email
  defp recipient(email, name), do: {name, email}
  defp recipient(%User{} = user), do: recipient(user.email, user.name)
--- a/lib/pleroma/emoji/formatter.ex
+++ b/lib/pleroma/emoji/formatter.ex
@ -5,7 +5,7 @@
 defmodule Pleroma.Emoji.Formatter do
  alias Pleroma.Emoji
  alias Pleroma.HTML
-  alias Pleroma.Web
+  alias Pleroma.Web.Endpoint
  alias Pleroma.Web.MediaProxy

  def emojify(text) do
@ -44,7 +44,7 @@ defmodule Pleroma.Emoji.Formatter do
    Emoji.get_all()
    |> Enum.filter(fn {emoji, %Emoji{}} -> String.contains?(text, ":#{emoji}:") end)
    |> Enum.reduce(%{}, fn {name, %Emoji{file: file}}, acc ->
-      Map.put(acc, name, to_string(URI.merge(Web.base_url(), file)))
+      Map.put(acc, name, to_string(URI.merge(Endpoint.url(), file)))
    end)
  end

--- a/lib/pleroma/formatter.ex
+++ b/lib/pleroma/formatter.ex
@ -62,7 +62,7 @@ defmodule Pleroma.Formatter do

  def hashtag_handler("#" <> tag = tag_text, _buffer, _opts, acc) do
    tag = String.downcase(tag)
-    url = "#{Pleroma.Web.base_url()}/tag/#{tag}"
+    url = "#{Pleroma.Web.Endpoint.url()}/tag/#{tag}"

    link =
      Phoenix.HTML.Tag.content_tag(:a, tag_text,
@ -121,6 +121,10 @@ defmodule Pleroma.Formatter do
    end
  end

+  def markdown_to_html(text) do
+    Earmark.as_html!(text, %Earmark.Options{compact_output: true})
+  end
+
  def html_escape({text, mentions, hashtags}, type) do
    {html_escape(text, type), mentions, hashtags}
  end
--- a/lib/pleroma/gun.ex
+++ b/lib/pleroma/gun.ex
@ -11,9 +11,7 @@ defmodule Pleroma.Gun do
  @callback await(pid(), reference()) :: {:response, :fin, 200, []}
  @callback set_owner(pid(), pid()) :: :ok

-  @api Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API)
-
-  defp api, do: @api
+  defp api, do: Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API)

  def open(host, port, opts), do: api().open(host, port, opts)

--- a/lib/pleroma/gun/connection_pool/reclaimer.ex
+++ b/lib/pleroma/gun/connection_pool/reclaimer.ex
@ -5,11 +5,11 @@
 defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
  use GenServer, restart: :temporary

-  @registry Pleroma.Gun.ConnectionPool
+  defp registry, do: Pleroma.Gun.ConnectionPool

  def start_monitor do
    pid =
-      case :gen_server.start(__MODULE__, [], name: {:via, Registry, {@registry, "reclaimer"}}) do
+      case :gen_server.start(__MODULE__, [], name: {:via, Registry, {registry(), "reclaimer"}}) do
        {:ok, pid} ->
          pid

@ -46,7 +46,7 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
    #   {worker_pid, crf, last_reference} end)
    unused_conns =
      Registry.select(
-        @registry,
+        registry(),
        [
          {{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}], [{{:"$1", :"$3", :"$4"}}]}
        ]
--- a/lib/pleroma/gun/connection_pool/worker.ex
+++ b/lib/pleroma/gun/connection_pool/worker.ex
@ -6,10 +6,10 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
  alias Pleroma.Gun
  use GenServer, restart: :temporary

-  @registry Pleroma.Gun.ConnectionPool
+  defp registry, do: Pleroma.Gun.ConnectionPool

  def start_link([key | _] = opts) do
-    GenServer.start_link(__MODULE__, opts, name: {:via, Registry, {@registry, key}})
+    GenServer.start_link(__MODULE__, opts, name: {:via, Registry, {registry(), key}})
  end

  @impl true
@ -24,7 +24,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
      time = :erlang.monotonic_time(:millisecond)

      {_, _} =
-        Registry.update_value(@registry, key, fn _ ->
+        Registry.update_value(registry(), key, fn _ ->
          {conn_pid, [client_pid], 1, time}
        end)

@ -65,7 +65,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
    time = :erlang.monotonic_time(:millisecond)

    {{conn_pid, used_by, _, _}, _} =
-      Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} ->
+      Registry.update_value(registry(), key, fn {conn_pid, used_by, crf, last_reference} ->
        {conn_pid, [client_pid | used_by], crf(time - last_reference, crf), time}
      end)

@ -92,7 +92,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
  @impl true
  def handle_call(:remove_client, {client_pid, _}, %{key: key} = state) do
    {{_conn_pid, used_by, _crf, _last_reference}, _} =
-      Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} ->
+      Registry.update_value(registry(), key, fn {conn_pid, used_by, crf, last_reference} ->
        {conn_pid, List.delete(used_by, client_pid), crf, last_reference}
      end)

--- a/lib/pleroma/hashtag.ex
+++ b/lib/pleroma/hashtag.ex
@ -0,0 +1,106 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Hashtag do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+  import Ecto.Query
+
+  alias Ecto.Multi
+  alias Pleroma.Hashtag
+  alias Pleroma.Object
+  alias Pleroma.Repo
+
+  schema "hashtags" do
+    field(:name, :string)
+
+    many_to_many(:objects, Object, join_through: "hashtags_objects", on_replace: :delete)
+
+    timestamps()
+  end
+
+  def normalize_name(name) do
+    name
+    |> String.downcase()
+    |> String.trim()
+  end
+
+  def get_or_create_by_name(name) do
+    changeset = changeset(%Hashtag{}, %{name: name})
+
+    Repo.insert(
+      changeset,
+      on_conflict: [set: [name: get_field(changeset, :name)]],
+      conflict_target: :name,
+      returning: true
+    )
+  end
+
+  def get_or_create_by_names(names) when is_list(names) do
+    names = Enum.map(names, &normalize_name/1)
+    timestamp = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second)
+
+    structs =
+      Enum.map(names, fn name ->
+        %Hashtag{}
+        |> changeset(%{name: name})
+        |> Map.get(:changes)
+        |> Map.merge(%{inserted_at: timestamp, updated_at: timestamp})
+      end)
+
+    try do
+      with {:ok, %{query_op: hashtags}} <-
+             Multi.new()
+             |> Multi.insert_all(:insert_all_op, Hashtag, structs,
+               on_conflict: :nothing,
+               conflict_target: :name
+             )
+             |> Multi.run(:query_op, fn _repo, _changes ->
+               {:ok, Repo.all(from(ht in Hashtag, where: ht.name in ^names))}
+             end)
+             |> Repo.transaction() do
+        {:ok, hashtags}
+      else
+        {:error, _name, value, _changes_so_far} -> {:error, value}
+      end
+    rescue
+      e -> {:error, e}
+    end
+  end
+
+  def changeset(%Hashtag{} = struct, params) do
+    struct
+    |> cast(params, [:name])
+    |> update_change(:name, &normalize_name/1)
+    |> validate_required([:name])
+    |> unique_constraint(:name)
+  end
+
+  def unlink(%Object{id: object_id}) do
+    with {_, hashtag_ids} <-
+           from(hto in "hashtags_objects",
+             where: hto.object_id == ^object_id,
+             select: hto.hashtag_id
+           )
+           |> Repo.delete_all(),
+         {:ok, unreferenced_count} <- delete_unreferenced(hashtag_ids) do
+      {:ok, length(hashtag_ids), unreferenced_count}
+    end
+  end
+
+  @delete_unreferenced_query """
+  DELETE FROM hashtags WHERE id IN
+    (SELECT hashtags.id FROM hashtags
+      LEFT OUTER JOIN hashtags_objects
+        ON hashtags_objects.hashtag_id = hashtags.id
+      WHERE hashtags_objects.hashtag_id IS NULL AND hashtags.id = ANY($1));
+  """
+
+  def delete_unreferenced(ids) do
+    with {:ok, %{num_rows: deleted_count}} <- Repo.query(@delete_unreferenced_query, [ids]) do
+      {:ok, deleted_count}
+    end
+  end
+end
--- a/lib/pleroma/html.ex
+++ b/lib/pleroma/html.ex
@ -49,31 +49,6 @@ defmodule Pleroma.HTML do
  def filter_tags(html), do: filter_tags(html, nil)
  def strip_tags(html), do: filter_tags(html, FastSanitize.Sanitizer.StripTags)

-  def get_cached_scrubbed_html_for_activity(
-        content,
-        scrubbers,
-        activity,
-        key \\ "",
-        callback \\ fn x -> x end
-      ) do
-    key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}"
-
-    @cachex.fetch!(:scrubber_cache, key, fn _key ->
-      object = Pleroma.Object.normalize(activity, fetch: false)
-      ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback)
-    end)
-  end
-
-  def get_cached_stripped_html_for_activity(content, activity, key) do
-    get_cached_scrubbed_html_for_activity(
-      content,
-      FastSanitize.Sanitizer.StripTags,
-      activity,
-      key,
-      &HtmlEntities.decode/1
-    )
-  end
-
  def ensure_scrubbed_html(
        content,
        scrubbers,
@ -92,16 +67,6 @@ defmodule Pleroma.HTML do
    end
  end

-  defp generate_scrubber_signature(scrubber) when is_atom(scrubber) do
-    generate_scrubber_signature([scrubber])
-  end
-
-  defp generate_scrubber_signature(scrubbers) do
-    Enum.reduce(scrubbers, "", fn scrubber, signature ->
-      "#{signature}#{to_string(scrubber)}"
-    end)
-  end
-
  def extract_first_external_url_from_object(%{data: %{"content" => content}} = object)
      when is_binary(content) do
    unless object.data["fake"] do
--- a/lib/pleroma/http/adapter_helper/gun.ex
+++ b/lib/pleroma/http/adapter_helper/gun.ex
@ -54,8 +54,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
    Config.get([:pools, pool, :recv_timeout], default)
  end

-  @prefix Pleroma.Gun.ConnectionPool
  def limiter_setup do
+    prefix = Pleroma.Gun.ConnectionPool
    wait = Config.get([:connections_pool, :connection_acquisition_wait])
    retries = Config.get([:connections_pool, :connection_acquisition_retries])

@ -66,7 +66,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
      max_waiting = Keyword.get(opts, :max_waiting, 10)

      result =
-        ConcurrentLimiter.new(:"#{@prefix}.#{name}", max_running, max_waiting,
+        ConcurrentLimiter.new(:"#{prefix}.#{name}", max_running, max_waiting,
          wait: wait,
          max_retries: retries
        )
--- a/lib/pleroma/http/web_push.ex
+++ b/lib/pleroma/http/web_push.ex
@ -5,8 +5,8 @@
 defmodule Pleroma.HTTP.WebPush do
  @moduledoc false

-  def post(url, payload, headers) do
+  def post(url, payload, headers, options \\ []) do
    list_headers = Map.to_list(headers)
-    Pleroma.HTTP.post(url, payload, list_headers)
+    Pleroma.HTTP.post(url, payload, list_headers, options)
  end
 end
--- a/lib/pleroma/instances.ex
+++ b/lib/pleroma/instances.ex
@ -5,13 +5,18 @@
 defmodule Pleroma.Instances do
  @moduledoc "Instances context."

-  @adapter Pleroma.Instances.Instance
+  alias Pleroma.Instances.Instance

-  defdelegate filter_reachable(urls_or_hosts), to: @adapter
-  defdelegate reachable?(url_or_host), to: @adapter
-  defdelegate set_reachable(url_or_host), to: @adapter
-  defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: @adapter
-  defdelegate get_consistently_unreachable(), to: @adapter
+  def filter_reachable(urls_or_hosts), do: Instance.filter_reachable(urls_or_hosts)
+
+  def reachable?(url_or_host), do: Instance.reachable?(url_or_host)
+
+  def set_reachable(url_or_host), do: Instance.set_reachable(url_or_host)
+
+  def set_unreachable(url_or_host, unreachable_since \\ nil),
+    do: Instance.set_unreachable(url_or_host, unreachable_since)
+
+  def get_consistently_unreachable, do: Instance.get_consistently_unreachable()

  def set_consistently_unreachable(url_or_host),
    do: set_unreachable(url_or_host, reachability_datetime_threshold())
--- a/lib/pleroma/maps.ex
+++ b/lib/pleroma/maps.ex
@ -12,4 +12,10 @@ defmodule Pleroma.Maps do
      _ -> map
    end
  end
+
+  def safe_put_in(data, keys, value) when is_map(data) and is_list(keys) do
+    Kernel.put_in(data, keys, value)
+  rescue
+    _ -> data
+  end
 end
--- a/lib/pleroma/migrators/hashtags_table_migrator.ex
+++ b/lib/pleroma/migrators/hashtags_table_migrator.ex
@ -0,0 +1,208 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Migrators.HashtagsTableMigrator do
+  defmodule State do
+    use Pleroma.Migrators.Support.BaseMigratorState
+
+    @impl Pleroma.Migrators.Support.BaseMigratorState
+    defdelegate data_migration(), to: Pleroma.DataMigration, as: :populate_hashtags_table
+  end
+
+  use Pleroma.Migrators.Support.BaseMigrator
+
+  alias Pleroma.Hashtag
+  alias Pleroma.Migrators.Support.BaseMigrator
+  alias Pleroma.Object
+
+  @impl BaseMigrator
+  def feature_config_path, do: [:features, :improved_hashtag_timeline]
+
+  @impl BaseMigrator
+  def fault_rate_allowance, do: Config.get([:populate_hashtags_table, :fault_rate_allowance], 0)
+
+  @impl BaseMigrator
+  def perform do
+    data_migration_id = data_migration_id()
+    max_processed_id = get_stat(:max_processed_id, 0)
+
+    Logger.info("Transferring embedded hashtags to `hashtags` (from oid: #{max_processed_id})...")
+
+    query()
+    |> where([object], object.id > ^max_processed_id)
+    |> Repo.chunk_stream(100, :batches, timeout: :infinity)
+    |> Stream.each(fn objects ->
+      object_ids = Enum.map(objects, & &1.id)
+
+      results = Enum.map(objects, &transfer_object_hashtags(&1))
+
+      failed_ids =
+        results
+        |> Enum.filter(&(elem(&1, 0) == :error))
+        |> Enum.map(&elem(&1, 1))
+
+      # Count of objects with hashtags: `{:noop, id}` is returned for objects having other AS2 tags
+      chunk_affected_count =
+        results
+        |> Enum.filter(&(elem(&1, 0) == :ok))
+        |> length()
+
+      for failed_id <- failed_ids do
+        _ =
+          Repo.query(
+            "INSERT INTO data_migration_failed_ids(data_migration_id, record_id) " <>
+              "VALUES ($1, $2) ON CONFLICT DO NOTHING;",
+            [data_migration_id, failed_id]
+          )
+      end
+
+      _ =
+        Repo.query(
+          "DELETE FROM data_migration_failed_ids " <>
+            "WHERE data_migration_id = $1 AND record_id = ANY($2)",
+          [data_migration_id, object_ids -- failed_ids]
+        )
+
+      max_object_id = Enum.at(object_ids, -1)
+
+      put_stat(:max_processed_id, max_object_id)
+      increment_stat(:iteration_processed_count, length(object_ids))
+      increment_stat(:processed_count, length(object_ids))
+      increment_stat(:failed_count, length(failed_ids))
+      increment_stat(:affected_count, chunk_affected_count)
+      put_stat(:records_per_second, records_per_second())
+      persist_state()
+
+      # A quick and dirty approach to controlling the load this background migration imposes
+      sleep_interval = Config.get([:populate_hashtags_table, :sleep_interval_ms], 0)
+      Process.sleep(sleep_interval)
+    end)
+    |> Stream.run()
+  end
+
+  @impl BaseMigrator
+  def query do
+    # Note: most objects have Mention-type AS2 tags and no hashtags (but we can't filter them out)
+    # Note: not checking activity type, expecting remove_non_create_objects_hashtags/_ to clean up
+    from(
+      object in Object,
+      where:
+        fragment("(?)->'tag' IS NOT NULL AND (?)->'tag' != '[]'::jsonb", object.data, object.data),
+      select: %{
+        id: object.id,
+        tag: fragment("(?)->'tag'", object.data)
+      }
+    )
+    |> join(:left, [o], hashtags_objects in fragment("SELECT object_id FROM hashtags_objects"),
+      on: hashtags_objects.object_id == o.id
+    )
+    |> where([_o, hashtags_objects], is_nil(hashtags_objects.object_id))
+  end
+
+  @spec transfer_object_hashtags(Map.t()) :: {:noop | :ok | :error, integer()}
+  defp transfer_object_hashtags(object) do
+    embedded_tags = if Map.has_key?(object, :tag), do: object.tag, else: object.data["tag"]
+    hashtags = Object.object_data_hashtags(%{"tag" => embedded_tags})
+
+    if Enum.any?(hashtags) do
+      transfer_object_hashtags(object, hashtags)
+    else
+      {:noop, object.id}
+    end
+  end
+
+  defp transfer_object_hashtags(object, hashtags) do
+    Repo.transaction(fn ->
+      with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do
+        maps = Enum.map(hashtag_records, &%{hashtag_id: &1.id, object_id: object.id})
+        base_error = "ERROR when inserting hashtags_objects for object with id #{object.id}"
+
+        try do
+          with {rows_count, _} when is_integer(rows_count) <-
+                 Repo.insert_all("hashtags_objects", maps, on_conflict: :nothing) do
+            object.id
+          else
+            e ->
+              Logger.error("#{base_error}: #{inspect(e)}")
+              Repo.rollback(object.id)
+          end
+        rescue
+          e ->
+            Logger.error("#{base_error}: #{inspect(e)}")
+            Repo.rollback(object.id)
+        end
+      else
+        e ->
+          error = "ERROR: could not create hashtags for object #{object.id}: #{inspect(e)}"
+          Logger.error(error)
+          Repo.rollback(object.id)
+      end
+    end)
+  end
+
+  @impl BaseMigrator
+  def retry_failed do
+    data_migration_id = data_migration_id()
+
+    failed_objects_query()
+    |> Repo.chunk_stream(100, :one)
+    |> Stream.each(fn object ->
+      with {res, _} when res != :error <- transfer_object_hashtags(object) do
+        _ =
+          Repo.query(
+            "DELETE FROM data_migration_failed_ids " <>
+              "WHERE data_migration_id = $1 AND record_id = $2",
+            [data_migration_id, object.id]
+          )
+      end
+    end)
+    |> Stream.run()
+
+    put_stat(:failed_count, failures_count())
+    persist_state()
+
+    force_continue()
+  end
+
+  defp failed_objects_query do
+    from(o in Object)
+    |> join(:inner, [o], dmf in fragment("SELECT * FROM data_migration_failed_ids"),
+      on: dmf.record_id == o.id
+    )
+    |> where([_o, dmf], dmf.data_migration_id == ^data_migration_id())
+    |> order_by([o], asc: o.id)
+  end
+
+  @doc """
+  Service func to delete `hashtags_objects` for legacy objects not associated with Create activity.
+  Also deletes unreferenced `hashtags` records (might occur after deletion of `hashtags_objects`).
+  """
+  def delete_non_create_activities_hashtags do
+    hashtags_objects_cleanup_query = """
+    DELETE FROM hashtags_objects WHERE object_id IN
+      (SELECT DISTINCT objects.id FROM objects
+        JOIN hashtags_objects ON hashtags_objects.object_id = objects.id LEFT JOIN activities
+          ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') =
+            (objects.data->>'id')
+          AND activities.data->>'type' = 'Create'
+        WHERE activities.id IS NULL);
+    """
+
+    hashtags_cleanup_query = """
+    DELETE FROM hashtags WHERE id IN
+      (SELECT hashtags.id FROM hashtags
+        LEFT OUTER JOIN hashtags_objects
+          ON hashtags_objects.hashtag_id = hashtags.id
+        WHERE hashtags_objects.hashtag_id IS NULL);
+    """
+
+    {:ok, %{num_rows: hashtags_objects_count}} =
+      Repo.query(hashtags_objects_cleanup_query, [], timeout: :infinity)
+
+    {:ok, %{num_rows: hashtags_count}} =
+      Repo.query(hashtags_cleanup_query, [], timeout: :infinity)
+
+    {:ok, hashtags_objects_count, hashtags_count}
+  end
+end
--- a/lib/pleroma/migrators/support/base_migrator.ex
+++ b/lib/pleroma/migrators/support/base_migrator.ex
@ -0,0 +1,210 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Migrators.Support.BaseMigrator do
+  @moduledoc """
+  Base background migrator functionality.
+  """
+
+  @callback perform() :: any()
+  @callback retry_failed() :: any()
+  @callback feature_config_path() :: list(atom())
+  @callback query() :: Ecto.Query.t()
+  @callback fault_rate_allowance() :: integer() | float()
+
+  defmacro __using__(_opts) do
+    quote do
+      use GenServer
+
+      require Logger
+
+      import Ecto.Query
+
+      alias __MODULE__.State
+      alias Pleroma.Config
+      alias Pleroma.Repo
+
+      @behaviour Pleroma.Migrators.Support.BaseMigrator
+
+      defdelegate data_migration(), to: State
+      defdelegate data_migration_id(), to: State
+      defdelegate state(), to: State
+      defdelegate persist_state(), to: State, as: :persist_to_db
+      defdelegate get_stat(key, value \\ nil), to: State, as: :get_data_key
+      defdelegate put_stat(key, value), to: State, as: :put_data_key
+      defdelegate increment_stat(key, increment), to: State, as: :increment_data_key
+
+      @reg_name {:global, __MODULE__}
+
+      def whereis, do: GenServer.whereis(@reg_name)
+
+      def start_link(_) do
+        case whereis() do
+          nil ->
+            GenServer.start_link(__MODULE__, nil, name: @reg_name)
+
+          pid ->
+            {:ok, pid}
+        end
+      end
+
+      @impl true
+      def init(_) do
+        {:ok, nil, {:continue, :init_state}}
+      end
+
+      @impl true
+      def handle_continue(:init_state, _state) do
+        {:ok, _} = State.start_link(nil)
+
+        data_migration = data_migration()
+        manual_migrations = Config.get([:instance, :manual_data_migrations], [])
+
+        cond do
+          Config.get(:env) == :test ->
+            update_status(:noop)
+
+          is_nil(data_migration) ->
+            message = "Data migration does not exist."
+            update_status(:failed, message)
+            Logger.error("#{__MODULE__}: #{message}")
+
+          data_migration.state == :manual or data_migration.name in manual_migrations ->
+            message = "Data migration is in manual execution or manual fix mode."
+            update_status(:manual, message)
+            Logger.warn("#{__MODULE__}: #{message}")
+
+          data_migration.state == :complete ->
+            on_complete(data_migration)
+
+          true ->
+            send(self(), :perform)
+        end
+
+        {:noreply, nil}
+      end
+
+      @impl true
+      def handle_info(:perform, state) do
+        State.reinit()
+
+        update_status(:running)
+        put_stat(:iteration_processed_count, 0)
+        put_stat(:started_at, NaiveDateTime.utc_now())
+
+        perform()
+
+        fault_rate = fault_rate()
+        put_stat(:fault_rate, fault_rate)
+        fault_rate_allowance = fault_rate_allowance()
+
+        cond do
+          fault_rate == 0 ->
+            set_complete()
+
+          is_float(fault_rate) and fault_rate <= fault_rate_allowance ->
+            message = """
+            Done with fault rate of #{fault_rate} which doesn't exceed #{fault_rate_allowance}.
+            Putting data migration to manual fix mode. Try running `#{__MODULE__}.retry_failed/0`.
+            """
+
+            Logger.warn("#{__MODULE__}: #{message}")
+            update_status(:manual, message)
+            on_complete(data_migration())
+
+          true ->
+            message = "Too many failures. Try running `#{__MODULE__}.retry_failed/0`."
+            Logger.error("#{__MODULE__}: #{message}")
+            update_status(:failed, message)
+        end
+
+        persist_state()
+        {:noreply, state}
+      end
+
+      defp on_complete(data_migration) do
+        if data_migration.feature_lock || feature_state() == :disabled do
+          Logger.warn(
+            "#{__MODULE__}: migration complete but feature is locked; consider enabling."
+          )
+
+          :noop
+        else
+          Config.put(feature_config_path(), :enabled)
+          :ok
+        end
+      end
+
+      @doc "Approximate count for current iteration (including processed records count)"
+      def count(force \\ false, timeout \\ :infinity) do
+        stored_count = get_stat(:count)
+
+        if stored_count && !force do
+          stored_count
+        else
+          processed_count = get_stat(:processed_count, 0)
+          max_processed_id = get_stat(:max_processed_id, 0)
+          query = where(query(), [entity], entity.id > ^max_processed_id)
+
+          count = Repo.aggregate(query, :count, :id, timeout: timeout) + processed_count
+          put_stat(:count, count)
+          persist_state()
+
+          count
+        end
+      end
+
+      def failures_count do
+        with {:ok, %{rows: [[count]]}} <-
+               Repo.query(
+                 "SELECT COUNT(record_id) FROM data_migration_failed_ids WHERE data_migration_id = $1;",
+                 [data_migration_id()]
+               ) do
+          count
+        end
+      end
+
+      def feature_state, do: Config.get(feature_config_path())
+
+      def force_continue do
+        send(whereis(), :perform)
+      end
+
+      def force_restart do
+        :ok = State.reset()
+        force_continue()
+      end
+
+      def set_complete do
+        update_status(:complete)
+        persist_state()
+        on_complete(data_migration())
+      end
+
+      defp update_status(status, message \\ nil) do
+        put_stat(:state, status)
+        put_stat(:message, message)
+      end
+
+      defp fault_rate do
+        with failures_count when is_integer(failures_count) <- failures_count() do
+          failures_count / Enum.max([get_stat(:affected_count, 0), 1])
+        else
+          _ -> :error
+        end
+      end
+
+      defp records_per_second do
+        get_stat(:iteration_processed_count, 0) / Enum.max([running_time(), 1])
+      end
+
+      defp running_time do
+        NaiveDateTime.diff(
+          NaiveDateTime.utc_now(),
+          get_stat(:started_at, NaiveDateTime.utc_now())
+        )
+      end
+    end
+  end
+end
--- a/lib/pleroma/migrators/support/base_migrator_state.ex
+++ b/lib/pleroma/migrators/support/base_migrator_state.ex
@ -0,0 +1,117 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Migrators.Support.BaseMigratorState do
+  @moduledoc """
+  Base background migrator state functionality.
+  """
+
+  @callback data_migration() :: Pleroma.DataMigration.t()
+
+  defmacro __using__(_opts) do
+    quote do
+      use Agent
+
+      alias Pleroma.DataMigration
+
+      @behaviour Pleroma.Migrators.Support.BaseMigratorState
+      @reg_name {:global, __MODULE__}
+
+      def start_link(_) do
+        Agent.start_link(fn -> load_state_from_db() end, name: @reg_name)
+      end
+
+      def data_migration, do: raise("data_migration/0 is not implemented")
+      defoverridable data_migration: 0
+
+      defp load_state_from_db do
+        data_migration = data_migration()
+
+        data =
+          if data_migration do
+            Map.new(data_migration.data, fn {k, v} -> {String.to_atom(k), v} end)
+          else
+            %{}
+          end
+
+        %{
+          data_migration_id: data_migration && data_migration.id,
+          data: data
+        }
+      end
+
+      def persist_to_db do
+        %{data_migration_id: data_migration_id, data: data} = state()
+
+        if data_migration_id do
+          DataMigration.update_one_by_id(data_migration_id, data: data)
+        else
+          {:error, :nil_data_migration_id}
+        end
+      end
+
+      def reset do
+        %{data_migration_id: data_migration_id} = state()
+
+        with false <- is_nil(data_migration_id),
+             :ok <-
+               DataMigration.update_one_by_id(data_migration_id,
+                 state: :pending,
+                 data: %{}
+               ) do
+          reinit()
+        else
+          true -> {:error, :nil_data_migration_id}
+          e -> e
+        end
+      end
+
+      def reinit do
+        Agent.update(@reg_name, fn _state -> load_state_from_db() end)
+      end
+
+      def state do
+        Agent.get(@reg_name, & &1)
+      end
+
+      def get_data_key(key, default \\ nil) do
+        get_in(state(), [:data, key]) || default
+      end
+
+      def put_data_key(key, value) do
+        _ = persist_non_data_change(key, value)
+
+        Agent.update(@reg_name, fn state ->
+          put_in(state, [:data, key], value)
+        end)
+      end
+
+      def increment_data_key(key, increment \\ 1) do
+        Agent.update(@reg_name, fn state ->
+          initial_value = get_in(state, [:data, key]) || 0
+          updated_value = initial_value + increment
+          put_in(state, [:data, key], updated_value)
+        end)
+      end
+
+      defp persist_non_data_change(:state, value) do
+        with true <- get_data_key(:state) != value,
+             true <- value in Pleroma.DataMigration.State.__valid_values__(),
+             %{data_migration_id: data_migration_id} when not is_nil(data_migration_id) <-
+               state() do
+          DataMigration.update_one_by_id(data_migration_id, state: value)
+        else
+          false -> :ok
+          _ -> {:error, :nil_data_migration_id}
+        end
+      end
+
+      defp persist_non_data_change(_, _) do
+        nil
+      end
+
+      def data_migration_id, do: Map.get(state(), :data_migration_id)
+    end
+  end
+end
--- a/lib/pleroma/object.ex
+++ b/lib/pleroma/object.ex
@ -10,6 +10,7 @@ defmodule Pleroma.Object do

  alias Pleroma.Activity
  alias Pleroma.Config
+  alias Pleroma.Hashtag
  alias Pleroma.Object
  alias Pleroma.Object.Fetcher
  alias Pleroma.ObjectTombstone
@ -28,6 +29,8 @@ defmodule Pleroma.Object do
  schema "objects" do
    field(:data, :map)

+    many_to_many(:hashtags, Hashtag, join_through: "hashtags_objects", on_replace: :delete)
+
    timestamps()
  end

@ -49,7 +52,8 @@ defmodule Pleroma.Object do
  end

  def create(data) do
-    Object.change(%Object{}, %{data: data})
+    %Object{}
+    |> Object.change(%{data: data})
    |> Repo.insert()
  end

@ -58,8 +62,41 @@ defmodule Pleroma.Object do
    |> cast(params, [:data])
    |> validate_required([:data])
    |> unique_constraint(:ap_id, name: :objects_unique_apid_index)
+    # Expecting `maybe_handle_hashtags_change/1` to run last:
+    |> maybe_handle_hashtags_change(struct)
  end

+  # Note: not checking activity type (assuming non-legacy objects are associated with Create act.)
+  defp maybe_handle_hashtags_change(changeset, struct) do
+    with %Ecto.Changeset{valid?: true} <- changeset,
+         data_hashtags_change = get_change(changeset, :data),
+         {_, true} <- {:changed, hashtags_changed?(struct, data_hashtags_change)},
+         {:ok, hashtag_records} <-
+           data_hashtags_change
+           |> object_data_hashtags()
+           |> Hashtag.get_or_create_by_names() do
+      put_assoc(changeset, :hashtags, hashtag_records)
+    else
+      %{valid?: false} ->
+        changeset
+
+      {:changed, false} ->
+        changeset
+
+      {:error, _} ->
+        validate_change(changeset, :data, fn _, _ ->
+          [data: "error referencing hashtags"]
+        end)
+    end
+  end
+
+  defp hashtags_changed?(%Object{} = struct, %{"tag" => _} = data) do
+    Enum.sort(embedded_hashtags(struct)) !=
+      Enum.sort(object_data_hashtags(data))
+  end
+
+  defp hashtags_changed?(_, _), do: false
+
  def get_by_id(nil), do: nil
  def get_by_id(id), do: Repo.get(Object, id)

@ -187,9 +224,13 @@ defmodule Pleroma.Object do
  def swap_object_with_tombstone(object) do
    tombstone = make_tombstone(object)

+    with {:ok, object} <-
           object
           |> Object.change(%{data: tombstone})
-    |> Repo.update()
+           |> Repo.update() do
+      Hashtag.unlink(object)
+      {:ok, object}
+    end
  end

  def delete(%Object{data: %{"id" => id}} = object) do
@ -325,7 +366,7 @@ defmodule Pleroma.Object do
  end

  def local?(%Object{data: %{"id" => id}}) do
-    String.starts_with?(id, Pleroma.Web.base_url() <> "/")
+    String.starts_with?(id, Pleroma.Web.Endpoint.url() <> "/")
  end

  def replies(object, opts \\ []) do
@ -349,4 +390,39 @@ defmodule Pleroma.Object do

  def self_replies(object, opts \\ []),
    do: replies(object, Keyword.put(opts, :self_only, true))
+
+  def tags(%Object{data: %{"tag" => tags}}) when is_list(tags), do: tags
+
+  def tags(_), do: []
+
+  def hashtags(%Object{} = object) do
+    # Note: always using embedded hashtags regardless whether they are migrated to hashtags table
+    #   (embedded hashtags stay in sync anyways, and we avoid extra joins and preload hassle)
+    embedded_hashtags(object)
+  end
+
+  def embedded_hashtags(%Object{data: data}) do
+    object_data_hashtags(data)
+  end
+
+  def embedded_hashtags(_), do: []
+
+  def object_data_hashtags(%{"tag" => tags}) when is_list(tags) do
+    tags
+    |> Enum.filter(fn
+      %{"type" => "Hashtag"} = data -> Map.has_key?(data, "name")
+      plain_text when is_bitstring(plain_text) -> true
+      _ -> false
+    end)
+    |> Enum.map(fn
+      %{"name" => "#" <> hashtag} -> String.downcase(hashtag)
+      %{"name" => hashtag} -> String.downcase(hashtag)
+      hashtag when is_bitstring(hashtag) -> String.downcase(hashtag)
+    end)
+    |> Enum.uniq()
+    # Note: "" elements (plain text) might occur in `data.tag` for incoming objects
+    |> Enum.filter(&(&1 not in [nil, ""]))
+  end
+
+  def object_data_hashtags(_), do: []
 end
--- a/lib/pleroma/object/containment.ex
+++ b/lib/pleroma/object/containment.ex
@ -71,6 +71,14 @@ defmodule Pleroma.Object.Containment do
    compare_uris(id_uri, other_uri)
  end

+  # Mastodon pin activities don't have an id, so we check the object field, which will be pinned.
+  def contain_origin_from_id(id, %{"object" => object}) when is_binary(object) do
+    id_uri = URI.parse(id)
+    object_uri = URI.parse(object)
+
+    compare_uris(id_uri, object_uri)
+  end
+
  def contain_origin_from_id(_id, _data), do: :error

  def contain_child(%{"object" => %{"id" => id, "attributedTo" => _} = object}),
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@ -4,6 +4,7 @@

 defmodule Pleroma.Object.Fetcher do
  alias Pleroma.HTTP
+  alias Pleroma.Maps
  alias Pleroma.Object
  alias Pleroma.Object.Containment
  alias Pleroma.Repo
@ -101,6 +102,9 @@ defmodule Pleroma.Object.Fetcher do
      {:transmogrifier, {:error, {:reject, e}}} ->
        {:reject, e}

+      {:transmogrifier, {:reject, e}} ->
+        {:reject, e}
+
      {:transmogrifier, _} = e ->
        {:error, e}

@ -124,12 +128,14 @@ defmodule Pleroma.Object.Fetcher do
  defp prepare_activity_params(data) do
    %{
      "type" => "Create",
-      "to" => data["to"] || [],
-      "cc" => data["cc"] || [],
      # Should we seriously keep this attributedTo thing?
      "actor" => data["actor"] || data["attributedTo"],
      "object" => data
    }
+    |> Maps.put_if_present("to", data["to"])
+    |> Maps.put_if_present("cc", data["cc"])
+    |> Maps.put_if_present("bto", data["bto"])
+    |> Maps.put_if_present("bcc", data["bcc"])
  end

  def fetch_object_from_id!(id, options \\ []) do
--- a/lib/pleroma/pagination.ex
+++ b/lib/pleroma/pagination.ex
@ -93,6 +93,7 @@ defmodule Pleroma.Pagination do
      max_id: :string,
      offset: :integer,
      limit: :integer,
+      skip_extra_order: :boolean,
      skip_order: :boolean
    }

@ -114,6 +115,8 @@ defmodule Pleroma.Pagination do

  defp restrict(query, :order, %{skip_order: true}, _), do: query

+  defp restrict(%{order_bys: [_ | _]} = query, :order, %{skip_extra_order: true}, _), do: query
+
  defp restrict(query, :order, %{min_id: _}, table_binding) do
    order_by(
      query,
--- a/lib/pleroma/repo.ex
+++ b/lib/pleroma/repo.ex
@ -8,6 +8,8 @@ defmodule Pleroma.Repo do
    adapter: Ecto.Adapters.Postgres,
    migration_timestamps: [type: :naive_datetime_usec]

+  use Ecto.Explain
+
  import Ecto.Query
  require Logger

@ -63,8 +65,8 @@ defmodule Pleroma.Repo do
  iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500, :batches)
  """
  @spec chunk_stream(Ecto.Query.t(), integer(), atom()) :: Enumerable.t()
-  def chunk_stream(query, chunk_size, returns_as \\ :one) do
-    # We don't actually need start and end funcitons of resource streaming,
+  def chunk_stream(query, chunk_size, returns_as \\ :one, query_options \\ []) do
+    # We don't actually need start and end functions of resource streaming,
    # but it seems to be the only way to not fetch records one-by-one and
    # have individual records be the elements of the stream, instead of
    # lists of records
@ -76,7 +78,7 @@ defmodule Pleroma.Repo do
          |> order_by(asc: :id)
          |> where([r], r.id > ^last_id)
          |> limit(^chunk_size)
-          |> all()
+          |> all(query_options)
          |> case do
            [] ->
              {:halt, last_id}
--- a/lib/pleroma/reverse_proxy.ex
+++ b/lib/pleroma/reverse_proxy.ex
@ -411,7 +411,7 @@ defmodule Pleroma.ReverseProxy do
    {:ok, :no_duration_limit, :no_duration_limit}
  end

-  defp client, do: Pleroma.ReverseProxy.Client
+  defp client, do: Pleroma.ReverseProxy.Client.Wrapper

  defp track_failed_url(url, error, opts) do
    ttl =
--- a/lib/pleroma/reverse_proxy/client.ex
+++ b/lib/pleroma/reverse_proxy/client.ex
@ -17,22 +17,4 @@ defmodule Pleroma.ReverseProxy.Client do
  @callback stream_body(map()) :: {:ok, binary(), map()} | :done | {:error, atom() | String.t()}

  @callback close(reference() | pid() | map()) :: :ok
-
-  def request(method, url, headers, body \\ "", opts \\ []) do
-    client().request(method, url, headers, body, opts)
-  end
-
-  def stream_body(ref), do: client().stream_body(ref)
-
-  def close(ref), do: client().close(ref)
-
-  defp client do
-    :tesla
-    |> Application.get_env(:adapter)
-    |> client()
-  end
-
-  defp client(Tesla.Adapter.Hackney), do: Pleroma.ReverseProxy.Client.Hackney
-  defp client(Tesla.Adapter.Gun), do: Pleroma.ReverseProxy.Client.Tesla
-  defp client(_), do: Pleroma.Config.get!(Pleroma.ReverseProxy.Client)
 end
--- a/lib/pleroma/reverse_proxy/client/wrapper.ex
+++ b/lib/pleroma/reverse_proxy/client/wrapper.ex
@ -0,0 +1,29 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ReverseProxy.Client.Wrapper do
+  @moduledoc "Meta-client that calls the appropriate client from the config."
+  @behaviour Pleroma.ReverseProxy.Client
+
+  @impl true
+  def request(method, url, headers, body \\ "", opts \\ []) do
+    client().request(method, url, headers, body, opts)
+  end
+
+  @impl true
+  def stream_body(ref), do: client().stream_body(ref)
+
+  @impl true
+  def close(ref), do: client().close(ref)
+
+  defp client do
+    :tesla
+    |> Application.get_env(:adapter)
+    |> client()
+  end
+
+  defp client(Tesla.Adapter.Hackney), do: Pleroma.ReverseProxy.Client.Hackney
+  defp client(Tesla.Adapter.Gun), do: Pleroma.ReverseProxy.Client.Tesla
+  defp client(_), do: Pleroma.Config.get!(Pleroma.ReverseProxy.Client)
+end
--- a/lib/pleroma/tests/auth_test_controller.ex
+++ b/lib/pleroma/tests/auth_test_controller.ex
@ -9,7 +9,6 @@ defmodule Pleroma.Tests.AuthTestController do
  use Pleroma.Web, :controller

  alias Pleroma.User
-  alias Pleroma.Web.Plugs.EnsurePublicOrAuthenticatedPlug
  alias Pleroma.Web.Plugs.OAuthScopesPlug

  # Serves only with proper OAuth token (:api and :authenticated_api)
@ -47,10 +46,7 @@ defmodule Pleroma.Tests.AuthTestController do
  # Via :authenticated_api, serves if token is present and has requested scopes
  #
  # Suggested use: as :fallback_oauth_check but open with nil :user for :api on private instances
-  plug(
-    :skip_plug,
-    EnsurePublicOrAuthenticatedPlug when action == :fallback_oauth_skip_publicity_check
-  )
+  plug(:skip_public_check when action == :fallback_oauth_skip_publicity_check)

  plug(
    OAuthScopesPlug,
@ -62,11 +58,7 @@ defmodule Pleroma.Tests.AuthTestController do
  # Via :authenticated_api, serves if :user is set (regardless of token presence and its scopes)
  #
  # Suggested use: making an :api endpoint always accessible (e.g. email confirmation endpoint)
-  plug(
-    :skip_plug,
-    [OAuthScopesPlug, EnsurePublicOrAuthenticatedPlug]
-    when action == :skip_oauth_skip_publicity_check
-  )
+  plug(:skip_auth when action == :skip_oauth_skip_publicity_check)

  # Via :authenticated_api, always fails with 403 (endpoint is insecure)
  # Via :api, drops :user if present and serves if public (private instance rejects on no user)
--- a/lib/pleroma/upload.ex
+++ b/lib/pleroma/upload.ex
@ -23,6 +23,9 @@ defmodule Pleroma.Upload do
    is once created permanent and changing it (especially in uploaders) is probably a bad idea!
  * `:tempfile` - path to the temporary file. Prefer in-place changes on the file rather than changing the
  path as the temporary file is also tracked by `Plug.Upload{}` and automatically deleted once the request is over.
+  * `:width` - width of the media in pixels
+  * `:height` - height of the media in pixels
+  * `:blurhash` - string hash of the image encoded with the blurhash algorithm (https://blurha.sh/)

  Related behaviors:

@ -32,6 +35,7 @@ defmodule Pleroma.Upload do
  """
  alias Ecto.UUID
  alias Pleroma.Config
+  alias Pleroma.Maps
  require Logger

  @type source ::
@ -53,9 +57,12 @@ defmodule Pleroma.Upload do
          name: String.t(),
          tempfile: String.t(),
          content_type: String.t(),
+          width: integer(),
+          height: integer(),
+          blurhash: String.t(),
          path: String.t()
        }
-  defstruct [:id, :name, :tempfile, :content_type, :path]
+  defstruct [:id, :name, :tempfile, :content_type, :width, :height, :blurhash, :path]

  defp get_description(opts, upload) do
    case {opts[:description], Pleroma.Config.get([Pleroma.Upload, :default_description])} do
@ -89,9 +96,12 @@ defmodule Pleroma.Upload do
             "mediaType" => upload.content_type,
             "href" => url_from_spec(upload, opts.base_url, url_spec)
           }
+           |> Maps.put_if_present("width", upload.width)
+           |> Maps.put_if_present("height", upload.height)
         ],
         "name" => description
-       }}
+       }
+       |> Maps.put_if_present("blurhash", upload.blurhash)}
    else
      {:description_limit, _} ->
        {:error, :description_too_long}
@ -225,7 +235,7 @@ defmodule Pleroma.Upload do

    case uploader do
      Pleroma.Uploaders.Local ->
-        upload_base_url || Pleroma.Web.base_url() <> "/media/"
+        upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/"

      Pleroma.Uploaders.S3 ->
        bucket = Config.get([Pleroma.Uploaders.S3, :bucket])
@ -251,7 +261,7 @@ defmodule Pleroma.Upload do
        end

      _ ->
-        public_endpoint || upload_base_url || Pleroma.Web.base_url() <> "/media/"
+        public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/"
    end
  end
 end
--- a/lib/pleroma/upload/filter.ex
+++ b/lib/pleroma/upload/filter.ex
@ -15,13 +15,13 @@ defmodule Pleroma.Upload.Filter do

  require Logger

-  @callback filter(Pleroma.Upload.t()) ::
+  @callback filter(upload :: struct()) ::
              {:ok, :filtered}
              | {:ok, :noop}
-              | {:ok, :filtered, Pleroma.Upload.t()}
+              | {:ok, :filtered, upload :: struct()}
              | {:error, any()}

-  @spec filter([module()], Pleroma.Upload.t()) :: {:ok, Pleroma.Upload.t()} | {:error, any()}
+  @spec filter([module()], upload :: struct()) :: {:ok, upload :: struct()} | {:error, any()}

  def filter([], upload) do
    {:ok, upload}
--- a/lib/pleroma/upload/filter/analyze_metadata.ex
+++ b/lib/pleroma/upload/filter/analyze_metadata.ex
@ -0,0 +1,83 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Upload.Filter.AnalyzeMetadata do
+  @moduledoc """
+  Extracts metadata about the upload, such as width/height
+  """
+  require Logger
+
+  @behaviour Pleroma.Upload.Filter
+
+  @spec filter(Pleroma.Upload.t()) ::
+          {:ok, :filtered, Pleroma.Upload.t()} | {:ok, :noop} | {:error, String.t()}
+  def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _} = upload) do
+    try do
+      image =
+        file
+        |> Mogrify.open()
+        |> Mogrify.verbose()
+
+      upload =
+        upload
+        |> Map.put(:width, image.width)
+        |> Map.put(:height, image.height)
+        |> Map.put(:blurhash, get_blurhash(file))
+
+      {:ok, :filtered, upload}
+    rescue
+      e in ErlangError ->
+        Logger.warn("#{__MODULE__}: #{inspect(e)}")
+        {:ok, :noop}
+    end
+  end
+
+  def filter(%Pleroma.Upload{tempfile: file, content_type: "video" <> _} = upload) do
+    try do
+      result = media_dimensions(file)
+
+      upload =
+        upload
+        |> Map.put(:width, result.width)
+        |> Map.put(:height, result.height)
+
+      {:ok, :filtered, upload}
+    rescue
+      e in ErlangError ->
+        Logger.warn("#{__MODULE__}: #{inspect(e)}")
+        {:ok, :noop}
+    end
+  end
+
+  def filter(_), do: {:ok, :noop}
+
+  defp get_blurhash(file) do
+    with {:ok, blurhash} <- :eblurhash.magick(file) do
+      blurhash
+    else
+      _ -> nil
+    end
+  end
+
+  defp media_dimensions(file) do
+    with executable when is_binary(executable) <- System.find_executable("ffprobe"),
+         args = [
+           "-v",
+           "error",
+           "-show_entries",
+           "stream=width,height",
+           "-of",
+           "csv=p=0:s=x",
+           file
+         ],
+         {result, 0} <- System.cmd(executable, args),
+         [width, height] <-
+           String.split(String.trim(result), "x") |> Enum.map(&String.to_integer(&1)) do
+      %{width: width, height: height}
+    else
+      nil -> {:error, {:ffprobe, :command_not_found}}
+      {:error, _} = error -> error
+    end
+  end
+end
--- a/lib/pleroma/uploaders/uploader.ex
+++ b/lib/pleroma/uploaders/uploader.ex
@ -35,7 +35,7 @@ defmodule Pleroma.Uploaders.Uploader do

  """
  @type file_spec :: {:file | :url, String.t()}
-  @callback put_file(Pleroma.Upload.t()) ::
+  @callback put_file(upload :: struct()) ::
              :ok | {:ok, file_spec()} | {:error, String.t()} | :wait_callback

  @callback delete_file(file :: String.t()) :: :ok | {:error, String.t()}
@ -46,7 +46,7 @@ defmodule Pleroma.Uploaders.Uploader do
              | {:error, Plug.Conn.t(), String.t()}
  @optional_callbacks http_callback: 2

-  @spec put_file(module(), Pleroma.Upload.t()) :: {:ok, file_spec()} | {:error, String.t()}
+  @spec put_file(module(), upload :: struct()) :: {:ok, file_spec()} | {:error, String.t()}
  def put_file(uploader, upload) do
    case uploader.put_file(upload) do
      :ok -> {:ok, {:file, upload.path}}
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@ -27,13 +27,13 @@ defmodule Pleroma.User do
  alias Pleroma.Repo
  alias Pleroma.User
  alias Pleroma.UserRelationship
-  alias Pleroma.Web
  alias Pleroma.Web.ActivityPub.ActivityPub
  alias Pleroma.Web.ActivityPub.Builder
  alias Pleroma.Web.ActivityPub.Pipeline
  alias Pleroma.Web.ActivityPub.Utils
  alias Pleroma.Web.CommonAPI
  alias Pleroma.Web.CommonAPI.Utils, as: CommonUtils
+  alias Pleroma.Web.Endpoint
  alias Pleroma.Web.OAuth
  alias Pleroma.Web.RelMe
  alias Pleroma.Workers.BackgroundWorker
@ -99,6 +99,7 @@ defmodule Pleroma.User do
    field(:local, :boolean, default: true)
    field(:follower_address, :string)
    field(:following_address, :string)
+    field(:featured_address, :string)
    field(:search_rank, :float, virtual: true)
    field(:search_type, :integer, virtual: true)
    field(:tags, {:array, :string}, default: [])
@ -130,7 +131,6 @@ defmodule Pleroma.User do
    field(:hide_followers, :boolean, default: false)
    field(:hide_follows, :boolean, default: false)
    field(:hide_favorites, :boolean, default: true)
-    field(:pinned_activities, {:array, :string}, default: [])
    field(:email_notifications, :map, default: %{"digest" => false})
    field(:mascot, :map, default: nil)
    field(:emoji, :map, default: %{})
@ -148,6 +148,7 @@ defmodule Pleroma.User do
    field(:accepts_chat_messages, :boolean, default: nil)
    field(:last_active_at, :naive_datetime)
    field(:disclose_client, :boolean, default: true)
+    field(:pinned_objects, :map, default: %{})

    embeds_one(
      :notification_settings,
@ -359,7 +360,7 @@ defmodule Pleroma.User do

      _ ->
        unless options[:no_default] do
-          Config.get([:assets, :default_user_avatar], "#{Web.base_url()}/images/avi.png")
+          Config.get([:assets, :default_user_avatar], "#{Endpoint.url()}/images/avi.png")
        end
    end
  end
@ -367,13 +368,15 @@ defmodule Pleroma.User do
  def banner_url(user, options \\ []) do
    case user.banner do
      %{"url" => [%{"href" => href} | _]} -> href
-      _ -> !options[:no_default] && "#{Web.base_url()}/images/banner.png"
+      _ -> !options[:no_default] && "#{Endpoint.url()}/images/banner.png"
    end
  end

  # Should probably be renamed or removed
-  def ap_id(%User{nickname: nickname}), do: "#{Web.base_url()}/users/#{nickname}"
+  @spec ap_id(User.t()) :: String.t()
+  def ap_id(%User{nickname: nickname}), do: "#{Endpoint.url()}/users/#{nickname}"

+  @spec ap_followers(User.t()) :: String.t()
  def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa
  def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers"

@ -381,6 +384,11 @@ defmodule Pleroma.User do
  def ap_following(%User{following_address: fa}) when is_binary(fa), do: fa
  def ap_following(%User{} = user), do: "#{ap_id(user)}/following"

+  @spec ap_featured_collection(User.t()) :: String.t()
+  def ap_featured_collection(%User{featured_address: fa}) when is_binary(fa), do: fa
+
+  def ap_featured_collection(%User{} = user), do: "#{ap_id(user)}/collections/featured"
+
  defp truncate_fields_param(params) do
    if Map.has_key?(params, :fields) do
      Map.put(params, :fields, Enum.map(params[:fields], &truncate_field/1))
@ -443,6 +451,7 @@ defmodule Pleroma.User do
        :uri,
        :follower_address,
        :following_address,
+        :featured_address,
        :hide_followers,
        :hide_follows,
        :hide_followers_count,
@ -454,7 +463,8 @@ defmodule Pleroma.User do
        :invisible,
        :actor_type,
        :also_known_as,
-        :accepts_chat_messages
+        :accepts_chat_messages,
+        :pinned_objects
      ]
    )
    |> cast(params, [:name], empty_values: [])
@ -686,7 +696,7 @@ defmodule Pleroma.User do
    |> validate_format(:nickname, local_nickname_regex())
    |> put_ap_id()
    |> unique_constraint(:ap_id)
-    |> put_following_and_follower_address()
+    |> put_following_and_follower_and_featured_address()
  end

  def register_changeset(struct, params \\ %{}, opts \\ []) do
@ -747,7 +757,7 @@ defmodule Pleroma.User do
    |> put_password_hash
    |> put_ap_id()
    |> unique_constraint(:ap_id)
-    |> put_following_and_follower_address()
+    |> put_following_and_follower_and_featured_address()
  end

  def maybe_validate_required_email(changeset, true), do: changeset
@ -765,11 +775,16 @@ defmodule Pleroma.User do
    put_change(changeset, :ap_id, ap_id)
  end

-  defp put_following_and_follower_address(changeset) do
-    followers = ap_followers(%User{nickname: get_field(changeset, :nickname)})
+  defp put_following_and_follower_and_featured_address(changeset) do
+    user = %User{nickname: get_field(changeset, :nickname)}
+    followers = ap_followers(user)
+    following = ap_following(user)
+    featured = ap_featured_collection(user)

    changeset
    |> put_change(:follower_address, followers)
+    |> put_change(:following_address, following)
+    |> put_change(:featured_address, featured)
  end

  defp autofollow_users(user) do
@ -2255,13 +2270,6 @@ defmodule Pleroma.User do
    |> update_and_set_cache()
  end

-  def roles(%{is_moderator: is_moderator, is_admin: is_admin}) do
-    %{
-      admin: is_admin,
-      moderator: is_moderator
-    }
-  end
-
  def validate_fields(changeset, remote? \\ false) do
    limit_name = if remote?, do: :max_remote_account_fields, else: :max_account_fields
    limit = Config.get([:instance, limit_name], 0)
@ -2350,45 +2358,35 @@ defmodule Pleroma.User do
    cast(user, %{is_approved: approved?}, [:is_approved])
  end

-  def add_pinnned_activity(user, %Pleroma.Activity{id: id}) do
-    if id not in user.pinned_activities do
-      max_pinned_statuses = Config.get([:instance, :max_pinned_statuses], 0)
-      params = %{pinned_activities: user.pinned_activities ++ [id]}
-
-      # if pinned activity was scheduled for deletion, we remove job
-      if expiration = Pleroma.Workers.PurgeExpiredActivity.get_expiration(id) do
-        Oban.cancel_job(expiration.id)
-      end
+  @spec add_pinned_object_id(User.t(), String.t()) :: {:ok, User.t()} | {:error, term()}
+  def add_pinned_object_id(%User{} = user, object_id) do
+    if !user.pinned_objects[object_id] do
+      params = %{pinned_objects: Map.put(user.pinned_objects, object_id, NaiveDateTime.utc_now())}

      user
-      |> cast(params, [:pinned_activities])
-      |> validate_length(:pinned_activities,
-        max: max_pinned_statuses,
-        message: "You have already pinned the maximum number of statuses"
-      )
+      |> cast(params, [:pinned_objects])
+      |> validate_change(:pinned_objects, fn :pinned_objects, pinned_objects ->
+        max_pinned_statuses = Config.get([:instance, :max_pinned_statuses], 0)
+
+        if Enum.count(pinned_objects) <= max_pinned_statuses do
+          []
+        else
+          [pinned_objects: "You have already pinned the maximum number of statuses"]
+        end
+      end)
    else
      change(user)
    end
    |> update_and_set_cache()
  end

-  def remove_pinnned_activity(user, %Pleroma.Activity{id: id, data: data}) do
-    params = %{pinned_activities: List.delete(user.pinned_activities, id)}
-
-    # if pinned activity was scheduled for deletion, we reschedule it for deletion
-    if data["expires_at"] do
-      # MRF.ActivityExpirationPolicy used UTC timestamps for expires_at in original implementation
-      {:ok, expires_at} =
-        data["expires_at"] |> Pleroma.EctoType.ActivityPub.ObjectValidators.DateTime.cast()
-
-      Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
-        activity_id: id,
-        expires_at: expires_at
-      })
-    end
-
+  @spec remove_pinned_object_id(User.t(), String.t()) :: {:ok, t()} | {:error, term()}
+  def remove_pinned_object_id(%User{} = user, object_id) do
    user
-    |> cast(params, [:pinned_activities])
+    |> cast(
+      %{pinned_objects: Map.delete(user.pinned_objects, object_id)},
+      [:pinned_objects]
+    )
    |> update_and_set_cache()
  end

--- a/lib/pleroma/user/query.ex
+++ b/lib/pleroma/user/query.ex
@ -27,7 +27,7 @@ defmodule Pleroma.User.Query do
      - e.g. Pleroma.User.Query.build(%{ap_id: ["http://ap_id1", "http://ap_id2"]})
  """
  import Ecto.Query
-  import Pleroma.Web.AdminAPI.Search, only: [not_empty_string: 1]
+  import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]

  alias Pleroma.FollowingRelationship
  alias Pleroma.User
--- a/lib/pleroma/utils.ex
+++ b/lib/pleroma/utils.ex
@ -11,6 +11,8 @@ defmodule Pleroma.Utils do
    eperm epipe erange erofs espipe esrch estale etxtbsy exdev
  )a

+  @repo_timeout Pleroma.Config.get([Pleroma.Repo, :timeout], 15_000)
+
  def compile_dir(dir) when is_binary(dir) do
    dir
    |> File.ls!()
@ -63,4 +65,21 @@ defmodule Pleroma.Utils do
  end

  def posix_error_message(_), do: ""
+
+  @doc """
+  Returns [timeout: integer] suitable for passing as an option to Repo functions.
+
+  This function detects if the execution was triggered from IEx shell, Mix task, or
+  ./bin/pleroma_ctl and sets the timeout to :infinity, else returns the default timeout value.
+  """
+  @spec query_timeout() :: [timeout: integer]
+  def query_timeout do
+    {parent, _, _, _} = Process.info(self(), :current_stacktrace) |> elem(1) |> Enum.fetch!(2)
+
+    cond do
+      parent |> to_string |> String.starts_with?("Elixir.Mix.Task") -> [timeout: :infinity]
+      parent == :erl_eval -> [timeout: :infinity]
+      true -> [timeout: @repo_timeout]
+    end
+  end
 end
--- a/lib/pleroma/web.ex
+++ b/lib/pleroma/web.ex
@ -35,9 +35,10 @@ defmodule Pleroma.Web do
      import Plug.Conn

      import Pleroma.Web.Gettext
-      import Pleroma.Web.Router.Helpers
      import Pleroma.Web.TranslationHelpers

+      alias Pleroma.Web.Router.Helpers, as: Routes
+
      plug(:set_put_layout)

      defp set_put_layout(conn, _) do
@ -61,6 +62,14 @@ defmodule Pleroma.Web do
        )
      end

+      defp skip_auth(conn, _) do
+        skip_plug(conn, [OAuthScopesPlug, EnsurePublicOrAuthenticatedPlug])
+      end
+
+      defp skip_public_check(conn, _) do
+        skip_plug(conn, EnsurePublicOrAuthenticatedPlug)
+      end
+
      # Executed just before actual controller action, invokes before-action hooks (callbacks)
      defp action(conn, params) do
        with %{halted: false} = conn <-
@ -131,7 +140,8 @@ defmodule Pleroma.Web do

      import Pleroma.Web.ErrorHelpers
      import Pleroma.Web.Gettext
-      import Pleroma.Web.Router.Helpers
+
+      alias Pleroma.Web.Router.Helpers, as: Routes

      require Logger

@ -229,20 +239,4 @@ defmodule Pleroma.Web do
  defmacro __using__(which) when is_atom(which) do
    apply(__MODULE__, which, [])
  end
-
-  def base_url do
-    Pleroma.Web.Endpoint.url()
-  end
-
-  # TODO: Change to Phoenix.Router.routes/1 for Phoenix 1.6.0+
-  def get_api_routes do
-    Pleroma.Web.Router.__routes__()
-    |> Enum.reject(fn r -> r.plug == Pleroma.Web.Fallback.RedirectController end)
-    |> Enum.map(fn r ->
-      r.path
-      |> String.split("/", trim: true)
-      |> List.first()
-    end)
-    |> Enum.uniq()
-  end
 end
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
  alias Pleroma.Conversation
  alias Pleroma.Conversation.Participation
  alias Pleroma.Filter
+  alias Pleroma.Hashtag
  alias Pleroma.Maps
  alias Pleroma.Notification
  alias Pleroma.Object
@ -87,7 +88,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do

  defp increase_replies_count_if_reply(_create_data), do: :noop

-  @object_types ~w[ChatMessage Question Answer Audio Video Event Article]
+  @object_types ~w[ChatMessage Question Answer Audio Video Event Article Note]
  @impl true
  def persist(%{"type" => type} = object, meta) when type in @object_types do
    with {:ok, object} <- Object.create(object) do
@ -465,6 +466,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
    |> Repo.one()
  end

+  defp fetch_paginated_optimized(query, opts, pagination) do
+    # Note: tag-filtering funcs may apply "ORDER BY objects.id DESC",
+    #   and extra sorting on "activities.id DESC NULLS LAST" would worse the query plan
+    opts = Map.put(opts, :skip_extra_order, true)
+
+    Pagination.fetch_paginated(query, opts, pagination)
+  end
+
+  def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do
+    list_memberships = Pleroma.List.memberships(opts[:user])
+
+    fetch_activities_query(recipients ++ list_memberships, opts)
+    |> fetch_paginated_optimized(opts, pagination)
+    |> Enum.reverse()
+    |> maybe_update_cc(list_memberships, opts[:user])
+  end
+
  @spec fetch_public_or_unlisted_activities(map(), Pagination.type()) :: [Activity.t()]
  def fetch_public_or_unlisted_activities(opts \\ %{}, pagination \\ :keyset) do
    opts = Map.delete(opts, :user)
@ -472,7 +490,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
    [Constants.as_public()]
    |> fetch_activities_query(opts)
    |> restrict_unlisted(opts)
-    |> Pagination.fetch_paginated(opts, pagination)
+    |> fetch_paginated_optimized(opts, pagination)
  end

  @spec fetch_public_activities(map(), Pagination.type()) :: [Activity.t()]
@ -612,7 +630,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
      |> Map.put(:type, ["Create", "Announce"])
      |> Map.put(:user, reading_user)
      |> Map.put(:actor_id, user.ap_id)
-      |> Map.put(:pinned_activity_ids, user.pinned_activities)
+      |> Map.put(:pinned_object_ids, Map.keys(user.pinned_objects))

    params =
      if User.blocks?(reading_user, user) do
@ -693,52 +711,144 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do

  defp restrict_since(query, _), do: query

-  defp restrict_tag_reject(_query, %{tag_reject: _tag_reject, skip_preload: true}) do
-    raise "Can't use the child object without preloading!"
+  defp restrict_embedded_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do
+    raise_on_missing_preload()
  end

-  defp restrict_tag_reject(query, %{tag_reject: [_ | _] = tag_reject}) do
-    from(
-      [_activity, object] in query,
-      where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject)
-    )
-  end
-
-  defp restrict_tag_reject(query, _), do: query
-
-  defp restrict_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do
-    raise "Can't use the child object without preloading!"
-  end
-
-  defp restrict_tag_all(query, %{tag_all: [_ | _] = tag_all}) do
+  defp restrict_embedded_tag_all(query, %{tag_all: [_ | _] = tag_all}) do
    from(
      [_activity, object] in query,
      where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all)
    )
  end

-  defp restrict_tag_all(query, _), do: query
+  defp restrict_embedded_tag_all(query, %{tag_all: tag}) when is_binary(tag) do
+    restrict_embedded_tag_any(query, %{tag: tag})
+  end

-  defp restrict_tag(_query, %{tag: _tag, skip_preload: true}) do
+  defp restrict_embedded_tag_all(query, _), do: query
+
+  defp restrict_embedded_tag_any(_query, %{tag: _tag, skip_preload: true}) do
+    raise_on_missing_preload()
+  end
+
+  defp restrict_embedded_tag_any(query, %{tag: [_ | _] = tag_any}) do
+    from(
+      [_activity, object] in query,
+      where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag_any)
+    )
+  end
+
+  defp restrict_embedded_tag_any(query, %{tag: tag}) when is_binary(tag) do
+    restrict_embedded_tag_any(query, %{tag: [tag]})
+  end
+
+  defp restrict_embedded_tag_any(query, _), do: query
+
+  defp restrict_embedded_tag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do
+    raise_on_missing_preload()
+  end
+
+  defp restrict_embedded_tag_reject_any(query, %{tag_reject: [_ | _] = tag_reject}) do
+    from(
+      [_activity, object] in query,
+      where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject)
+    )
+  end
+
+  defp restrict_embedded_tag_reject_any(query, %{tag_reject: tag_reject})
+       when is_binary(tag_reject) do
+    restrict_embedded_tag_reject_any(query, %{tag_reject: [tag_reject]})
+  end
+
+  defp restrict_embedded_tag_reject_any(query, _), do: query
+
+  defp object_ids_query_for_tags(tags) do
+    from(hto in "hashtags_objects")
+    |> join(:inner, [hto], ht in Pleroma.Hashtag, on: hto.hashtag_id == ht.id)
+    |> where([hto, ht], ht.name in ^tags)
+    |> select([hto], hto.object_id)
+    |> distinct([hto], true)
+  end
+
+  defp restrict_hashtag_all(_query, %{tag_all: _tag, skip_preload: true}) do
+    raise_on_missing_preload()
+  end
+
+  defp restrict_hashtag_all(query, %{tag_all: [single_tag]}) do
+    restrict_hashtag_any(query, %{tag: single_tag})
+  end
+
+  defp restrict_hashtag_all(query, %{tag_all: [_ | _] = tags}) do
+    from(
+      [_activity, object] in query,
+      where:
+        fragment(
+          """
+          (SELECT array_agg(hashtags.name) FROM hashtags JOIN hashtags_objects
+            ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?)
+              AND hashtags_objects.object_id = ?) @> ?
+          """,
+          ^tags,
+          object.id,
+          ^tags
+        )
+    )
+  end
+
+  defp restrict_hashtag_all(query, %{tag_all: tag}) when is_binary(tag) do
+    restrict_hashtag_all(query, %{tag_all: [tag]})
+  end
+
+  defp restrict_hashtag_all(query, _), do: query
+
+  defp restrict_hashtag_any(_query, %{tag: _tag, skip_preload: true}) do
+    raise_on_missing_preload()
+  end
+
+  defp restrict_hashtag_any(query, %{tag: [_ | _] = tags}) do
+    hashtag_ids =
+      from(ht in Hashtag, where: ht.name in ^tags, select: ht.id)
+      |> Repo.all()
+
+    # Note: NO extra ordering should be done on "activities.id desc nulls last" for optimal plan
+    from(
+      [_activity, object] in query,
+      join: hto in "hashtags_objects",
+      on: hto.object_id == object.id,
+      where: hto.hashtag_id in ^hashtag_ids,
+      distinct: [desc: object.id],
+      order_by: [desc: object.id]
+    )
+  end
+
+  defp restrict_hashtag_any(query, %{tag: tag}) when is_binary(tag) do
+    restrict_hashtag_any(query, %{tag: [tag]})
+  end
+
+  defp restrict_hashtag_any(query, _), do: query
+
+  defp restrict_hashtag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do
+    raise_on_missing_preload()
+  end
+
+  defp restrict_hashtag_reject_any(query, %{tag_reject: [_ | _] = tags_reject}) do
+    from(
+      [_activity, object] in query,
+      where: object.id not in subquery(object_ids_query_for_tags(tags_reject))
+    )
+  end
+
+  defp restrict_hashtag_reject_any(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do
+    restrict_hashtag_reject_any(query, %{tag_reject: [tag_reject]})
+  end
+
+  defp restrict_hashtag_reject_any(query, _), do: query
+
+  defp raise_on_missing_preload do
    raise "Can't use the child object without preloading!"
  end

-  defp restrict_tag(query, %{tag: tag}) when is_list(tag) do
-    from(
-      [_activity, object] in query,
-      where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag)
-    )
-  end
-
-  defp restrict_tag(query, %{tag: tag}) when is_binary(tag) do
-    from(
-      [_activity, object] in query,
-      where: fragment("(?)->'tag' \\? (?)", object.data, ^tag)
-    )
-  end
-
-  defp restrict_tag(query, _), do: query
-
  defp restrict_recipients(query, [], _user), do: query

  defp restrict_recipients(query, recipients, nil) do
@ -965,8 +1075,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do

  defp restrict_unlisted(query, _), do: query

-  defp restrict_pinned(query, %{pinned: true, pinned_activity_ids: ids}) do
-    from(activity in query, where: activity.id in ^ids)
+  defp restrict_pinned(query, %{pinned: true, pinned_object_ids: ids}) do
+    from(
+      [activity, object: o] in query,
+      where:
+        fragment(
+          "(?)->>'type' = 'Create' and coalesce((?)->'object'->>'id', (?)->>'object') = any (?)",
+          activity.data,
+          activity.data,
+          activity.data,
+          ^ids
+        )
+    )
  end

  defp restrict_pinned(query, _), do: query
@ -1098,6 +1218,26 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do

  defp maybe_order(query, _), do: query

+  defp normalize_fetch_activities_query_opts(opts) do
+    Enum.reduce([:tag, :tag_all, :tag_reject], opts, fn key, opts ->
+      case opts[key] do
+        value when is_bitstring(value) ->
+          Map.put(opts, key, Hashtag.normalize_name(value))
+
+        value when is_list(value) ->
+          normalized_value =
+            value
+            |> Enum.map(&Hashtag.normalize_name/1)
+            |> Enum.uniq()
+
+          Map.put(opts, key, normalized_value)
+
+        _ ->
+          opts
+      end
+    end)
+  end
+
  defp fetch_activities_query_ap_ids_ops(opts) do
    source_user = opts[:muting_user]
    ap_id_relationships = if source_user, do: [:mute, :reblog_mute], else: []
@ -1121,6 +1261,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
  end

  def fetch_activities_query(recipients, opts \\ %{}) do
+    opts = normalize_fetch_activities_query_opts(opts)
+
    {restrict_blocked_opts, restrict_muted_opts, restrict_muted_reblogs_opts} =
      fetch_activities_query_ap_ids_ops(opts)

@ -1128,6 +1270,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
      skip_thread_containment: Config.get([:instance, :skip_thread_containment])
    }

+    query =
      Activity
      |> maybe_preload_objects(opts)
      |> maybe_preload_bookmarks(opts)
@ -1136,9 +1279,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
      |> maybe_order(opts)
      |> restrict_recipients(recipients, opts[:user])
      |> restrict_replies(opts)
-    |> restrict_tag(opts)
-    |> restrict_tag_reject(opts)
-    |> restrict_tag_all(opts)
      |> restrict_since(opts)
      |> restrict_local(opts)
      |> restrict_remote(opts)
@ -1163,15 +1303,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
      |> exclude_chat_messages(opts)
      |> exclude_invisible_actors(opts)
      |> exclude_visibility(opts)
+
+    if Config.feature_enabled?(:improved_hashtag_timeline) do
+      query
+      |> restrict_hashtag_any(opts)
+      |> restrict_hashtag_all(opts)
+      |> restrict_hashtag_reject_any(opts)
+    else
+      query
+      |> restrict_embedded_tag_any(opts)
+      |> restrict_embedded_tag_all(opts)
+      |> restrict_embedded_tag_reject_any(opts)
    end
-
-  def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do
-    list_memberships = Pleroma.List.memberships(opts[:user])
-
-    fetch_activities_query(recipients ++ list_memberships, opts)
-    |> Pagination.fetch_paginated(opts, pagination)
-    |> Enum.reverse()
-    |> maybe_update_cc(list_memberships, opts[:user])
  end

  @doc """
@ -1250,21 +1393,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do

  defp get_actor_url(_url), do: nil

+  defp normalize_image(%{"url" => url}) do
+    %{
+      "type" => "Image",
+      "url" => [%{"href" => url}]
+    }
+  end
+
+  defp normalize_image(urls) when is_list(urls), do: urls |> List.first() |> normalize_image()
+  defp normalize_image(_), do: nil
+
  defp object_to_user_data(data) do
-    avatar =
-      data["icon"]["url"] &&
-        %{
-          "type" => "Image",
-          "url" => [%{"href" => data["icon"]["url"]}]
-        }
-
-    banner =
-      data["image"]["url"] &&
-        %{
-          "type" => "Image",
-          "url" => [%{"href" => data["image"]["url"]}]
-        }
-
    fields =
      data
      |> Map.get("attachment", [])
@ -1290,6 +1429,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
    invisible = data["invisible"] || false
    actor_type = data["type"] || "Person"

+    featured_address = data["featured"]
+    {:ok, pinned_objects} = fetch_and_prepare_featured_from_ap_id(featured_address)
+
    public_key =
      if is_map(data["publicKey"]) && is_binary(data["publicKey"]["publicKeyPem"]) do
        data["publicKey"]["publicKeyPem"]
@ -1308,23 +1450,25 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
      ap_id: data["id"],
      uri: get_actor_url(data["url"]),
      ap_enabled: true,
-      banner: banner,
+      banner: normalize_image(data["image"]),
      fields: fields,
      emoji: emojis,
      is_locked: is_locked,
      is_discoverable: is_discoverable,
      invisible: invisible,
-      avatar: avatar,
+      avatar: normalize_image(data["icon"]),
      name: data["name"],
      follower_address: data["followers"],
      following_address: data["following"],
+      featured_address: featured_address,
      bio: data["summary"] || "",
      actor_type: actor_type,
      also_known_as: Map.get(data, "alsoKnownAs", []),
      public_key: public_key,
      inbox: data["inbox"],
      shared_inbox: shared_inbox,
-      accepts_chat_messages: accepts_chat_messages
+      accepts_chat_messages: accepts_chat_messages,
+      pinned_objects: pinned_objects
    }

    # nickname can be nil because of virtual actors
@ -1462,6 +1606,41 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
    end
  end

+  def pin_data_from_featured_collection(%{
+        "type" => type,
+        "orderedItems" => objects
+      })
+      when type in ["OrderedCollection", "Collection"] do
+    Map.new(objects, fn %{"id" => object_ap_id} -> {object_ap_id, NaiveDateTime.utc_now()} end)
+  end
+
+  def fetch_and_prepare_featured_from_ap_id(nil) do
+    {:ok, %{}}
+  end
+
+  def fetch_and_prepare_featured_from_ap_id(ap_id) do
+    with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do
+      {:ok, pin_data_from_featured_collection(data)}
+    else
+      e ->
+        Logger.error("Could not decode featured collection at fetch #{ap_id}, #{inspect(e)}")
+        {:ok, %{}}
+    end
+  end
+
+  def pinned_fetch_task(nil), do: nil
+
+  def pinned_fetch_task(%{pinned_objects: pins}) do
+    if Enum.all?(pins, fn {ap_id, _} ->
+         Object.get_cached_by_ap_id(ap_id) ||
+           match?({:ok, _object}, Fetcher.fetch_object_from_id(ap_id))
+       end) do
+      :ok
+    else
+      :error
+    end
+  end
+
  def make_user_from_ap_id(ap_id) do
    user = User.get_cached_by_ap_id(ap_id)

@ -1469,6 +1648,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
      Transmogrifier.upgrade_user_from_ap_id(ap_id)
    else
      with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id) do
+        {:ok, _pid} = Task.start(fn -> pinned_fetch_task(data) end)
+
        if user do
          user
          |> User.remote_user_changeset(data)
--- a/lib/pleroma/web/activity_pub/activity_pub/persisting.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub/persisting.ex
@ -3,5 +3,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only

 defmodule Pleroma.Web.ActivityPub.ActivityPub.Persisting do
-  @callback persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
+  @callback persist(map(), keyword()) :: {:ok, struct()}
 end
--- a/lib/pleroma/web/activity_pub/activity_pub/streaming.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub/streaming.ex
@ -3,10 +3,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only

 defmodule Pleroma.Web.ActivityPub.ActivityPub.Streaming do
-  alias Pleroma.Activity
-  alias Pleroma.Object
-  alias Pleroma.User
-
-  @callback stream_out(Activity.t()) :: any()
-  @callback stream_out_participations(Object.t(), User.t()) :: any()
+  @callback stream_out(struct()) :: any()
+  @callback stream_out_participations(struct(), struct()) :: any()
 end
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@ -543,4 +543,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
      |> json(object.data)
    end
  end
+
+  def pinned(conn, %{"nickname" => nickname}) do
+    with %User{} = user <- User.get_cached_by_nickname(nickname) do
+      conn
+      |> put_resp_header("content-type", "application/activity+json")
+      |> json(UserView.render("featured.json", %{user: user}))
+    end
+  end
 end
--- a/lib/pleroma/web/activity_pub/builder.ex
+++ b/lib/pleroma/web/activity_pub/builder.ex
@ -223,7 +223,7 @@ defmodule Pleroma.Web.ActivityPub.Builder do
          [actor.follower_address]

        public? and Visibility.is_local_public?(object) ->
-          [actor.follower_address, object.data["actor"], Pleroma.Constants.as_local_public()]
+          [actor.follower_address, object.data["actor"], Utils.as_local_public()]

        public? ->
          [actor.follower_address, object.data["actor"], Pleroma.Constants.as_public()]
@ -273,4 +273,36 @@ defmodule Pleroma.Web.ActivityPub.Builder do
       "context" => object.data["context"]
     }, []}
  end
+
+  @spec pin(User.t(), Object.t()) :: {:ok, map(), keyword()}
+  def pin(%User{} = user, object) do
+    {:ok,
+     %{
+       "id" => Utils.generate_activity_id(),
+       "target" => pinned_url(user.nickname),
+       "object" => object.data["id"],
+       "actor" => user.ap_id,
+       "type" => "Add",
+       "to" => [Pleroma.Constants.as_public()],
+       "cc" => [user.follower_address]
+     }, []}
+  end
+
+  @spec unpin(User.t(), Object.t()) :: {:ok, map, keyword()}
+  def unpin(%User{} = user, object) do
+    {:ok,
+     %{
+       "id" => Utils.generate_activity_id(),
+       "target" => pinned_url(user.nickname),
+       "object" => object.data["id"],
+       "actor" => user.ap_id,
+       "type" => "Remove",
+       "to" => [Pleroma.Constants.as_public()],
+       "cc" => [user.follower_address]
+     }, []}
+  end
+
+  defp pinned_url(nickname) when is_binary(nickname) do
+    Pleroma.Web.Router.Helpers.activity_pub_url(Pleroma.Web.Endpoint, :pinned, nickname)
+  end
 end
--- a/lib/pleroma/web/activity_pub/mrf.ex
+++ b/lib/pleroma/web/activity_pub/mrf.ex
@ -51,17 +51,6 @@ defmodule Pleroma.Web.ActivityPub.MRF do

  @required_description_keys [:key, :related_policy]

-  @callback filter(Map.t()) :: {:ok | :reject, Map.t()}
-  @callback describe() :: {:ok | :error, Map.t()}
-  @callback config_description() :: %{
-              optional(:children) => [map()],
-              key: atom(),
-              related_policy: String.t(),
-              label: String.t(),
-              description: String.t()
-            }
-  @optional_callbacks config_description: 0
-
  def filter(policies, %{} = message) do
    policies
    |> Enum.reduce({:ok, message}, fn
@ -92,7 +81,9 @@ defmodule Pleroma.Web.ActivityPub.MRF do
  end

  def get_policies do
-    Pleroma.Config.get([:mrf, :policies], []) |> get_policies()
+    Pleroma.Config.get([:mrf, :policies], [])
+    |> get_policies()
+    |> Enum.concat([Pleroma.Web.ActivityPub.MRF.HashtagPolicy])
  end

  defp get_policies(policy) when is_atom(policy), do: [policy]
@ -140,7 +131,7 @@ defmodule Pleroma.Web.ActivityPub.MRF do
  def describe, do: get_policies() |> describe()

  def config_descriptions do
-    Pleroma.Web.ActivityPub.MRF
+    Pleroma.Web.ActivityPub.MRF.Policy
    |> Pleroma.Docs.Generator.list_behaviour_implementations()
    |> config_descriptions()
  end
--- a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
@ -4,7 +4,7 @@

 defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy do
  @moduledoc "Adds expiration to all local Create activities"
-  @behaviour Pleroma.Web.ActivityPub.MRF
+  @behaviour Pleroma.Web.ActivityPub.MRF.Policy

  @impl true
  def filter(activity) do
--- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex
@ -7,7 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do

  @moduledoc "Prevent followbots from following with a bit of heuristic"

-  @behaviour Pleroma.Web.ActivityPub.MRF
+  @behaviour Pleroma.Web.ActivityPub.MRF.Policy

  # XXX: this should become User.normalize_by_ap_id() or similar, really.
  defp normalize_by_ap_id(%{"id" => id}), do: User.get_cached_by_ap_id(id)
--- a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex
@ -5,7 +5,7 @@
 defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do
  alias Pleroma.User

-  @behaviour Pleroma.Web.ActivityPub.MRF
+  @behaviour Pleroma.Web.ActivityPub.MRF.Policy

  require Logger

--- a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex
@ -5,7 +5,7 @@
 defmodule Pleroma.Web.ActivityPub.MRF.DropPolicy do
  require Logger
  @moduledoc "Drop and log everything received"
-  @behaviour Pleroma.Web.ActivityPub.MRF
+  @behaviour Pleroma.Web.ActivityPub.MRF.Policy

  @impl true
  def filter(object) do
--- a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex
+++ b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex
@ -6,7 +6,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do
  alias Pleroma.Object

  @moduledoc "Ensure a re: is prepended on replies to a post with a Subject"
-  @behaviour Pleroma.Web.ActivityPub.MRF
+  @behaviour Pleroma.Web.ActivityPub.MRF.Policy

  @reply_prefix Regex.compile!("^re:[[:space:]]*", [:caseless])

--- a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex
@ -0,0 +1,59 @@
+defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do
+  @behaviour Pleroma.Web.ActivityPub.MRF.Policy
+  alias Pleroma.Config
+  alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
+
+  require Logger
+
+  @impl true
+  def filter(message) do
+    with follower_nickname <- Config.get([:mrf_follow_bot, :follower_nickname]),
+         %User{actor_type: "Service"} = follower <-
+           User.get_cached_by_nickname(follower_nickname),
+         %{"type" => "Create", "object" => %{"type" => "Note"}} <- message do
+      try_follow(follower, message)
+    else
+      nil ->
+        Logger.warn(
+          "#{__MODULE__} skipped because of missing `:mrf_follow_bot, :follower_nickname` configuration, the :follower_nickname
+            account does not exist, or the account is not correctly configured as a bot."
+        )
+
+        {:ok, message}
+
+      _ ->
+        {:ok, message}
+    end
+  end
+
+  defp try_follow(follower, message) do
+    to = Map.get(message, "to", [])
+    cc = Map.get(message, "cc", [])
+    actor = [message["actor"]]
+
+    Enum.concat([to, cc, actor])
+    |> List.flatten()
+    |> Enum.uniq()
+    |> User.get_all_by_ap_id()
+    |> Enum.each(fn user ->
+      with false <- user.local,
+           false <- User.following?(follower, user),
+           false <- User.locked?(user),
+           false <- (user.bio || "") |> String.downcase() |> String.contains?("nobot") do
+        Logger.debug(
+          "#{__MODULE__}: Follow request from #{follower.nickname} to #{user.nickname}"
+        )
+
+        CommonAPI.follow(follower, user)
+      end
+    end)
+
+    {:ok, message}
+  end
+
+  @impl true
+  def describe do
+    {:ok, %{}}
+  end
+end
--- a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex
@ -4,7 +4,7 @@

 defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy do
  alias Pleroma.User
-  @behaviour Pleroma.Web.ActivityPub.MRF
+  @behaviour Pleroma.Web.ActivityPub.MRF.Policy
  @moduledoc "Remove bot posts from federated timeline"

  require Pleroma.Constants
--- a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex
@ -0,0 +1,116 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do
+  require Pleroma.Constants
+
+  alias Pleroma.Config
+  alias Pleroma.Object
+
+  @moduledoc """
+  Reject, TWKN-remove or Set-Sensitive messsages with specific hashtags (without the leading #)
+
+  Note: This MRF Policy is always enabled, if you want to disable it you have to set empty lists.
+  """
+
+  @behaviour Pleroma.Web.ActivityPub.MRF.Policy
+
+  defp check_reject(message, hashtags) do
+    if Enum.any?(Config.get([:mrf_hashtag, :reject]), fn match -> match in hashtags end) do
+      {:reject, "[HashtagPolicy] Matches with rejected keyword"}
+    else
+      {:ok, message}
+    end
+  end
+
+  defp check_ftl_removal(%{"to" => to} = message, hashtags) do
+    if Pleroma.Constants.as_public() in to and
+         Enum.any?(Config.get([:mrf_hashtag, :federated_timeline_removal]), fn match ->
+           match in hashtags
+         end) do
+      to = List.delete(to, Pleroma.Constants.as_public())
+      cc = [Pleroma.Constants.as_public() | message["cc"] || []]
+
+      message =
+        message
+        |> Map.put("to", to)
+        |> Map.put("cc", cc)
+        |> Kernel.put_in(["object", "to"], to)
+        |> Kernel.put_in(["object", "cc"], cc)
+
+      {:ok, message}
+    else
+      {:ok, message}
+    end
+  end
+
+  defp check_ftl_removal(message, _hashtags), do: {:ok, message}
+
+  defp check_sensitive(message, hashtags) do
+    if Enum.any?(Config.get([:mrf_hashtag, :sensitive]), fn match -> match in hashtags end) do
+      {:ok, Kernel.put_in(message, ["object", "sensitive"], true)}
+    else
+      {:ok, message}
+    end
+  end
+
+  @impl true
+  def filter(%{"type" => "Create", "object" => object} = message) do
+    hashtags = Object.hashtags(%Object{data: object})
+
+    if hashtags != [] do
+      with {:ok, message} <- check_reject(message, hashtags),
+           {:ok, message} <- check_ftl_removal(message, hashtags),
+           {:ok, message} <- check_sensitive(message, hashtags) do
+        {:ok, message}
+      end
+    else
+      {:ok, message}
+    end
+  end
+
+  @impl true
+  def filter(message), do: {:ok, message}
+
+  @impl true
+  def describe do
+    mrf_hashtag =
+      Config.get(:mrf_hashtag)
+      |> Enum.into(%{})
+
+    {:ok, %{mrf_hashtag: mrf_hashtag}}
+  end
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_hashtag,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.HashtagPolicy",
+      label: "MRF Hashtag",
+      description: @moduledoc,
+      children: [
+        %{
+          key: :reject,
+          type: {:list, :string},
+          description: "A list of hashtags which result in message being rejected.",
+          suggestions: ["foo"]
+        },
+        %{
+          key: :federated_timeline_removal,
+          type: {:list, :string},
+          description:
+            "A list of hashtags which result in message being removed from federated timelines (a.k.a unlisted).",
+          suggestions: ["foo"]
+        },
+        %{
+          key: :sensitive,
+          type: {:list, :string},
+          description:
+            "A list of hashtags which result in message being set as sensitive (a.k.a NSFW/R-18)",
+          suggestions: ["nsfw", "r18"]
+        }
+      ]
+    }
+  end
+end
--- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex
@ -9,7 +9,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do

  @moduledoc "Block messages with too much mentions (configurable)"

-  @behaviour Pleroma.Web.ActivityPub.MRF
+  @behaviour Pleroma.Web.ActivityPub.MRF.Policy

  defp delist_message(message, threshold) when threshold > 0 do
    follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address
--- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex
@ -7,7 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do

  @moduledoc "Reject or Word-Replace messages with a keyword or regex"

-  @behaviour Pleroma.Web.ActivityPub.MRF
+  @behaviour Pleroma.Web.ActivityPub.MRF.Policy
  defp string_matches?(string, _) when not is_binary(string) do
    false
  end
--- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
@ -4,7 +4,7 @@

 defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
  @moduledoc "Preloads any attachments in the MediaProxy cache by prefetching them"
-  @behaviour Pleroma.Web.ActivityPub.MRF
+  @behaviour Pleroma.Web.ActivityPub.MRF.Policy

  alias Pleroma.HTTP
  alias Pleroma.Web.MediaProxy
--- a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex
@ -5,7 +5,7 @@
 defmodule Pleroma.Web.ActivityPub.MRF.MentionPolicy do
  @moduledoc "Block messages which mention a user"

-  @behaviour Pleroma.Web.ActivityPub.MRF
+  @behaviour Pleroma.Web.ActivityPub.MRF.Policy

  @impl true
  def filter(%{"type" => "Create"} = message) do
--- a/Show More
+++ b/Show More