Changelog: Add info and warning about object ap_ids.

CHANGELOG.md

@@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Authentication: Added rate limit for password-authorized actions / login existence checks
 
 ### Changed
+- **ATTENTION - LONG MIGRATION** Objects have an ap_id column now, extracted with a trigger.
 - **Breaking:** Elixir >=1.8 is now required (was >= 1.7)
 - **Breaking:** Admin API: Return link alongside with token on password reset
 - Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings)

lib/pleroma/activity.ex

@@ -70,8 +70,8 @@ defmodule Pleroma.Activity do
     join(query, join_type, [activity], o in Object,
       on:
         fragment(
-          "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
-          o.data,
+          "? = COALESCE(?->'object'->>'id', ?->>'object')",
+          o.ap_id,
           activity.data,
           activity.data
         ),

lib/pleroma/object.ex

@@ -19,13 +19,21 @@ defmodule Pleroma.Object do
 
   schema "objects" do
     field(:data, :map)
+    # This is set by a database side trigger on insert and update
+    field(:ap_id, :string)
 
     timestamps()
   end
 
+  # Add the default 'returning' options, so we get the generated ap_id column
+  def insert(cng, options \\ []) do
+    cng
+    |> Repo.insert(Keyword.put(options, :returning, true))
+  end
+
   def create(data) do
     Object.change(%Object{}, %{data: data})
-    |> Repo.insert()
+    |> insert()
   end
 
   def change(struct, params \\ %{}) do
@@ -59,7 +67,7 @@ defmodule Pleroma.Object do
   def get_by_ap_id(nil), do: nil
 
   def get_by_ap_id(ap_id) do
-    Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id)))
+    Repo.one(from(object in Object, where: object.ap_id == ^ap_id))
   end
 
   defp warn_on_no_object_preloaded(ap_id) do

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -1013,7 +1013,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           data
         end
 
-      Repo.insert(%Object{data: obj_data})
+      Object.insert(%Object{data: obj_data})
     end
   end
 

lib/pleroma/web/activity_pub/utils.ex

@@ -143,7 +143,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
 
     with true <- is_nil(object),
          changeset <- Object.context_mapping(context),
-         {:ok, inserted_object} <- Repo.insert(changeset) do
+         {:ok, inserted_object} <- Object.insert(changeset) do
       inserted_object
     else
       _ ->

priv/repo/migrations/20191007123056_add_ap_id_column_to_objects.exs

@@ -0,0 +1,9 @@
+defmodule Pleroma.Repo.Migrations.AddApIdColumnToObjects do
+  use Ecto.Migration
+
+  def change do
+    alter table(:objects) do
+      add :ap_id, :string
+    end
+  end
+end

priv/repo/migrations/20191007160755_fill_object_ap_id_field.exs

@@ -0,0 +1,11 @@
+defmodule Pleroma.Repo.Migrations.FillObjectApIdField do
+  use Ecto.Migration
+  alias Pleroma.Clippy
+
+  def change do
+    Clippy.puts("ATTENTION! This migration might take hours! If you don't want to run it now, abort this with CTRL+C! I'll wait 30 seconds now.")
+    :timer.sleep(:timer.seconds(30))
+    execute("update objects set ap_id = data->>'id'")
+    create unique_index(:objects, [:ap_id])
+  end
+end

priv/repo/migrations/20191007170423_add_object_ap_id_trigger.exs

@@ -0,0 +1,25 @@
+defmodule Pleroma.Repo.Migrations.AddObjectApIdTrigger do
+  use Ecto.Migration
+
+  def change do
+    execute("""
+    CREATE OR REPLACE FUNCTION set_ap_id()
+    RETURNS trigger
+    LANGUAGE plpgsql
+    AS $BODY$
+    BEGIN
+      NEW.ap_id = NEW.data->>'id';
+      RETURN NEW;
+    END
+    $BODY$;
+    """)
+
+    execute("""
+    CREATE TRIGGER object_ap_id_extraction
+    BEFORE INSERT OR UPDATE
+    ON objects
+    FOR EACH ROW
+    EXECUTE PROCEDURE set_ap_id();
+    """)
+  end
+end

test/object_test.exs

@@ -19,6 +19,7 @@ defmodule Pleroma.ObjectTest do
 
   test "returns an object by it's AP id" do
     object = insert(:note)
+    object = Object.get_by_id(object.id)
     found_object = Object.get_by_ap_id(object.data["id"])
 
     assert object == found_object
@@ -32,11 +33,21 @@ defmodule Pleroma.ObjectTest do
 
       {:error, _result} = Repo.insert(cs)
     end
+
+    test "it automatically sets the ap_id field" do
+      {:ok, object} =
+        %Object{}
+        |> Object.change(%{data: %{id: "myid"}})
+        |> Object.insert()
+
+      assert object.ap_id == "myid"
+    end
   end
 
   describe "deletion function" do
     test "deletes an object" do
       object = insert(:note)
+      object = Object.get_by_id(object.id)
       found_object = Object.get_by_ap_id(object.data["id"])
 
       assert object == found_object
@@ -52,6 +63,7 @@ defmodule Pleroma.ObjectTest do
 
     test "ensures cache is cleared for the object" do
       object = insert(:note)
+      object = Object.get_by_id(object.id)
       cached_object = Object.get_cached_by_ap_id(object.data["id"])
 
       assert object == cached_object