From 709816a0f891d6c26c43b54577a3b727c1fe4af6 Mon Sep 17 00:00:00 2001 From: Thurloat Date: Mon, 27 Aug 2018 22:20:54 -0300 Subject: [PATCH] example of flexible storage backends --- config/config.exs | 10 +++++-- lib/pleroma/upload.ex | 66 ++---------------------------------------- lib/pleroma/uploaders/local.ex | 44 ++++++++++++++++++++++++++++ lib/pleroma/uploaders/s3.ex | 24 +++++++++++++++ lib/pleroma/uploaders/swift.ex | 0 5 files changed, 77 insertions(+), 67 deletions(-) create mode 100644 lib/pleroma/uploaders/local.ex create mode 100644 lib/pleroma/uploaders/s3.ex create mode 100644 lib/pleroma/uploaders/swift.ex diff --git a/config/config.exs b/config/config.exs index d234d23eb..4eb59994c 100644 --- a/config/config.exs +++ b/config/config.exs @@ -11,9 +11,13 @@ config :pleroma, ecto_repos: [Pleroma.Repo] config :pleroma, Pleroma.Repo, types: Pleroma.PostgresTypes config :pleroma, Pleroma.Upload, - uploads: "uploads", - strip_exif: false, - use_s3: false, + uploader: Pleroma.Uploaders.Local + strip_exif: false + +config :pleroma, Pleroma.Uploaders.Local, + uploads: "uploads" + +config :pleroma, Pleroma.Uploaders.S3, s3_bucket: nil config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"] diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index a744e6fd4..d7cc8122a 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -4,31 +4,15 @@ defmodule Pleroma.Upload do def store(%Plug.Upload{} = file, should_dedupe) do settings = Application.get_env(:pleroma, Pleroma.Upload) - use_s3 = Keyword.fetch!(settings, :use_s3) + storage_backend = Keyword.fetch!(settings, :storage_backend) content_type = get_content_type(file.path) uuid = get_uuid(file, should_dedupe) name = get_name(file, uuid, content_type, should_dedupe) - upload_folder = get_upload_path(uuid, should_dedupe) - url_path = get_url(name, uuid, should_dedupe) strip_exif_data(content_type, file.path) - File.mkdir_p!(upload_folder) - result_file = Path.join(upload_folder, name) - - if File.exists?(result_file) do - File.rm!(file.path) - else - File.cp!(file.path, result_file) - end - - url_path = - if use_s3 do - put_s3_file(name, uuid, result_file, content_type) - else - url_path - end + url_path = storage_backend.put_file(name, uuid, content_type) %{ "type" => "Document", @@ -115,11 +99,6 @@ defmodule Pleroma.Upload do end end - def upload_path do - settings = Application.get_env(:pleroma, Pleroma.Upload) - Keyword.fetch!(settings, :uploads) - end - defp create_name(uuid, ext, type) do case type do "application/octet-stream" -> @@ -163,26 +142,6 @@ defmodule Pleroma.Upload do end end - defp get_upload_path(uuid, should_dedupe) do - if should_dedupe do - upload_path() - else - Path.join(upload_path(), uuid) - end - end - - defp get_url(name, uuid, should_dedupe) do - if should_dedupe do - url_for(:cow_uri.urlencode(name)) - else - url_for(Path.join(uuid, :cow_uri.urlencode(name))) - end - end - - defp url_for(file) do - "#{Web.base_url()}/media/#{file}" - end - def get_content_type(file) do match = File.open(file, [:read], fn f -> @@ -224,25 +183,4 @@ defmodule Pleroma.Upload do _e -> "application/octet-stream" end end - - defp put_s3_file(name, uuid, path, content_type) do - settings = Application.get_env(:pleroma, Pleroma.Upload) - bucket = Keyword.fetch!(settings, :bucket) - public_endpoint = Keyword.fetch!(settings, :public_endpoint) - - {:ok, file_data} = File.read(path) - - File.rm!(path) - - s3_name = "#{uuid}/#{name}" - - {:ok, result} = - ExAws.S3.put_object(bucket, s3_name, file_data, [ - {:acl, :public_read}, - {:content_type, content_type} - ]) - |> ExAws.request() - - "#{public_endpoint}/#{bucket}/#{s3_name}" - end end diff --git a/lib/pleroma/uploaders/local.ex b/lib/pleroma/uploaders/local.ex new file mode 100644 index 000000000..7e5d7a59b --- /dev/null +++ b/lib/pleroma/uploaders/local.ex @@ -0,0 +1,44 @@ +defmodule Pleroma.Uploaders.Local do + def put_file(name, uuid, file, content_type) do + + upload_path = get_upload_path(uuid, should_dedupe) + url_path = get_url(name, uuid, should_dedupe) + + File.mkdir_p!(upload_folder) + + result_file = Path.join(upload_folder, name) + + if File.exists?(result_file) do + File.rm!(file.path) + else + File.cp!(file.path, result_file) + end + + url_path + end + + def upload_path do + settings = Application.get_env(:pleroma, Pleroma.Uploaders.Local) + Keyword.fetch!(settings, :uploads) + end + + defp get_upload_path(uuid, should_dedupe) do + if should_dedupe do + upload_path() + else + Path.join(upload_path(), uuid) + end + end + + defp get_url(name, uuid, should_dedupe) do + if should_dedupe do + url_for(:cow_uri.urlencode(name)) + else + url_for(Path.join(uuid, :cow_uri.urlencode(name))) + end + end + + defp url_for(file) do + "#{Web.base_url()}/media/#{file}" + end +end diff --git a/lib/pleroma/uploaders/s3.ex b/lib/pleroma/uploaders/s3.ex new file mode 100644 index 000000000..95f20be67 --- /dev/null +++ b/lib/pleroma/uploaders/s3.ex @@ -0,0 +1,24 @@ +defmodule Pleroma.Uploaders.S3 do + + def put_file(name, uuid, path, content_type) do + + settings = Application.get_env(:pleroma, Pleroma.Uploaders.S3) + bucket = Keyword.fetch!(settings, :bucket) + public_endpoint = Keyword.fetch!(settings, :public_endpoint) + + {:ok, file_data} = File.read(path) + + File.rm!(path) + + s3_name = "#{uuid}/#{name}" + + {:ok, result} = + ExAws.S3.put_object(bucket, s3_name, file_data, [ + {:acl, :public_read}, + {:content_type, content_type} + ]) + |> ExAws.request() + + "#{public_endpoint}/#{bucket}/#{s3_name}" + end +end diff --git a/lib/pleroma/uploaders/swift.ex b/lib/pleroma/uploaders/swift.ex new file mode 100644 index 000000000..e69de29bb