From 0d521022fe6157ce9a346c6915ce38292e653bb3 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 7 Jan 2021 12:20:29 +0300
Subject: [PATCH] [#3213] Removed PK from hashtags_objects table. Improved
 hashtags_transfer mix task (logging of failed ids).

---
 lib/mix/tasks/pleroma/database.ex                  | 29 +++++++++++++---------
 lib/pleroma/object.ex                              |  4 +--
 .../20201221203824_create_hashtags_objects.exs     |  2 +-
 3 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index f903cf75b..918752dc2 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -139,6 +139,7 @@ defmodule Mix.Tasks.Pleroma.Database do
 
     Logger.info("Starting transferring object embedded hashtags to `hashtags` table...")
 
+    # Note: most objects have Mention-type AS2 tags and no hashtags (but we can't filter them out)
     from(
       object in Object,
       left_join: hashtag in assoc(object, :hashtags),
@@ -153,13 +154,10 @@ defmodule Mix.Tasks.Pleroma.Database do
     |> Stream.each(fn objects ->
       Logger.info("Processing #{length(objects)} objects starting from id #{hd(objects).id}...")
 
-      Enum.map(
-        objects,
-        fn object ->
-          hashtags =
-            object.tag
-            |> Jason.decode!()
-            |> Enum.filter(&is_bitstring(&1))
+      failed_ids =
+        objects
+        |> Enum.map(fn object ->
+          hashtags = Object.object_data_hashtags(%{"tag" => Jason.decode!(object.tag)})
 
           Repo.transaction(fn ->
             with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do
@@ -169,7 +167,7 @@ defmodule Mix.Tasks.Pleroma.Database do
                          "insert into hashtags_objects(hashtag_id, object_id) values ($1, $2);",
                          [hashtag_record.id, object.id]
                        ) do
-                  :noop
+                  nil
                 else
                   {:error, e} ->
                     error =
@@ -177,18 +175,25 @@ defmodule Mix.Tasks.Pleroma.Database do
                         "#{hashtag_record.id}: #{inspect(e)}"
 
                     Logger.error(error)
-                    Repo.rollback(error)
+                    Repo.rollback(object.id)
                 end
               end
+
+              object.id
             else
               e ->
                 error = "ERROR: could not create hashtags for object #{object.id}: #{inspect(e)}"
                 Logger.error(error)
-                Repo.rollback(error)
+                Repo.rollback(object.id)
             end
           end)
-        end
-      )
+        end)
+        |> Enum.filter(&(elem(&1, 0) == :error))
+        |> Enum.map(&elem(&1, 1))
+
+      if Enum.any?(failed_ids) do
+        Logger.error("ERROR: transfer_hashtags iteration failed for ids: #{inspect(failed_ids)}")
+      end
     end)
     |> Stream.run()
 
diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
index 7e79e15ee..61f2ffa19 100644
--- a/lib/pleroma/object.ex
+++ b/lib/pleroma/object.ex
@@ -404,7 +404,7 @@ defmodule Pleroma.Object do
 
   defp embedded_hashtags(_), do: []
 
-  defp object_data_hashtags(%{"tag" => tags}) when is_list(tags) do
+  def object_data_hashtags(%{"tag" => tags}) when is_list(tags) do
     tags
     |> Enum.filter(fn
       %{"type" => "Hashtag"} = data -> Map.has_key?(data, "name")
@@ -419,5 +419,5 @@ defmodule Pleroma.Object do
     |> Enum.uniq()
   end
 
-  defp object_data_hashtags(_), do: []
+  def object_data_hashtags(_), do: []
 end
diff --git a/priv/repo/migrations/20201221203824_create_hashtags_objects.exs b/priv/repo/migrations/20201221203824_create_hashtags_objects.exs
index b2649b4fb..214ea81c3 100644
--- a/priv/repo/migrations/20201221203824_create_hashtags_objects.exs
+++ b/priv/repo/migrations/20201221203824_create_hashtags_objects.exs
@@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateHashtagsObjects do
   use Ecto.Migration
 
   def change do
-    create_if_not_exists table(:hashtags_objects) do
+    create_if_not_exists table(:hashtags_objects, primary_key: false) do
       add(:hashtag_id, references(:hashtags), null: false)
       add(:object_id, references(:objects), null: false)
     end