From 6a42641b8d806f40f697303995fb12af39a93bd8 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 10 Aug 2019 21:46:36 +0300 Subject: Add pack.toml loading --- lib/pleroma/emoji.ex | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 66e20f0e4..ede734a53 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -143,23 +143,38 @@ defmodule Pleroma.Emoji do defp load_pack(pack_dir, emoji_groups) do pack_name = Path.basename(pack_dir) - emoji_txt = Path.join(pack_dir, "emoji.txt") + pack_toml = Path.join(pack_dir, "pack.toml") - if File.exists?(emoji_txt) do - load_from_file(emoji_txt, emoji_groups) + if File.exists?(pack_toml) do + toml = Toml.decode_file!(pack_toml) + + toml["files"] + |> Enum.map(fn {name, rel_file} -> + filename = Path.join("/emoji/#{pack_name}", rel_file) + {name, filename, pack_name} + end) else - extensions = Pleroma.Config.get([:emoji, :pack_extensions]) + # Load from emoji.txt / all files + emoji_txt = Path.join(pack_dir, "emoji.txt") - Logger.info( - "No emoji.txt found for pack \"#{pack_name}\", assuming all #{Enum.join(extensions, ", ")} files are emoji" - ) + if File.exists?(emoji_txt) do + load_from_file(emoji_txt, emoji_groups) + else + extensions = Pleroma.Config.get([:emoji, :pack_extensions]) - make_shortcode_to_file_map(pack_dir, extensions) - |> Enum.map(fn {shortcode, rel_file} -> - filename = Path.join("/emoji/#{pack_name}", rel_file) + Logger.info( + "No emoji.txt found for pack \"#{pack_name}\", assuming all #{ + Enum.join(extensions, ", ") + } files are emoji" + ) - {shortcode, filename, [to_string(match_extra(emoji_groups, filename))]} - end) + make_shortcode_to_file_map(pack_dir, extensions) + |> Enum.map(fn {shortcode, rel_file} -> + filename = Path.join("/emoji/#{pack_name}", rel_file) + + {shortcode, filename, [to_string(match_extra(emoji_groups, filename))]} + end) + end end end -- cgit v1.2.3 From b791a0865641eb8210380e22e04a9fb680a79dcb Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 11 Aug 2019 00:39:21 +0300 Subject: Implement API actions on packs That incldues listing them and downloading them from other instances or from the remote url --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 171 ++++++++++++++++++++++ lib/pleroma/web/router.ex | 22 +++ 2 files changed, 193 insertions(+) create mode 100644 lib/pleroma/web/emoji_api/emoji_api_controller.ex (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex new file mode 100644 index 000000000..49d671518 --- /dev/null +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -0,0 +1,171 @@ +defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do + use Pleroma.Web, :controller + + def reload(conn, _params) do + Pleroma.Emoji.reload() + + conn |> json("ok") + end + + @emoji_dir_path Path.join( + Pleroma.Config.get!([:instance, :static_dir]), + "emoji" + ) + + def list_packs(conn, _params) do + pack_infos = + case File.ls(@emoji_dir_path) do + {:error, _} -> + %{} + + {:ok, results} -> + results + |> Enum.filter(fn file -> + dir_path = Path.join(@emoji_dir_path, file) + # Filter to only use the pack.toml packs + File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.toml")) + end) + |> Enum.map(fn pack_name -> + pack_path = Path.join(@emoji_dir_path, pack_name) + pack_file = Path.join(pack_path, "pack.toml") + + {pack_name, Toml.decode_file!(pack_file)} + end) + # Transform into a map of pack-name => pack-data + # Check if all the files are in place and can be sent + |> Enum.map(fn {name, pack} -> + pack_path = Path.join(@emoji_dir_path, name) + + archive_for_sha = make_archive(name, pack, pack_path) + archive_sha = :crypto.hash(:sha256, archive_for_sha) |> Base.encode16() + + {name, + pack + |> put_in(["pack", "can-download"], can_download?(pack, pack_path)) + |> put_in(["pack", "download-sha256"], archive_sha)} + end) + |> Enum.into(%{}) + end + + conn |> json(pack_infos) + end + + defp can_download?(pack, pack_path) do + # If the pack is set as shared, check if it can be downloaded + # That means that when asked, the pack can be packed and sent to the remote + # Otherwise, they'd have to download it from external-src + pack["pack"]["share-files"] and + Enum.all?(pack["files"], fn {_, path} -> + File.exists?(Path.join(pack_path, path)) + end) + end + + defp make_archive(name, pack, pack_dir) do + files = + ['pack.toml'] ++ + (pack["files"] |> Enum.map(fn {_, path} -> to_charlist(path) end)) + + {:ok, {_, zip_result}} = :zip.zip('#{name}.zip', files, [:memory, cwd: to_charlist(pack_dir)]) + + zip_result + end + + def download_shared(conn, %{"name" => name}) do + pack_dir = Path.join(@emoji_dir_path, name) + pack_toml = Path.join(pack_dir, "pack.toml") + + if File.exists?(pack_toml) do + pack = Toml.decode_file!(pack_toml) + + if can_download?(pack, pack_dir) do + zip_result = make_archive(name, pack, pack_dir) + + conn + |> send_download({:binary, zip_result}, filename: "#{name}.zip") + else + {:error, + conn + |> put_status(:forbidden) + |> json("Pack #{name} cannot be downloaded from this instance, either pack sharing\ + was disabled for this pack or some files are missing")} + end + else + {:error, + conn + |> put_status(:not_found) + |> json("Pack #{name} does not exist")} + end + end + + def download_from(conn, %{"instance_address" => address, "pack_name" => name} = data) do + list_uri = "#{address}/api/pleroma/emoji/packs/list" + + list = Tesla.get!(list_uri).body |> Jason.decode!() + full_pack = list[name] + pfiles = full_pack["files"] + pack = full_pack["pack"] + + pack_info_res = + cond do + pack["share-files"] && pack["can-download"] -> + {:ok, + %{ + sha: pack["download-sha256"], + uri: "#{address}/api/pleroma/emoji/packs/download_shared/#{name}" + }} + + pack["fallback-src"] -> + {:ok, + %{ + sha: pack["fallback-src-sha256"], + uri: pack["fallback-src"], + fallback: true + }} + + true -> + {:error, "The pack was not set as shared and the is no fallback url to download from"} + end + + case pack_info_res do + {:ok, %{sha: sha, uri: uri} = pinfo} -> + sha = Base.decode16!(sha) + emoji_archive = Tesla.get!(uri).body + + got_sha = :crypto.hash(:sha256, emoji_archive) + + if got_sha == sha do + local_name = data["as"] || name + pack_dir = Path.join(@emoji_dir_path, local_name) + File.mkdir_p!(pack_dir) + + files = + ['pack.toml'] ++ + (pfiles |> Enum.map(fn {_, path} -> to_charlist(path) end)) + + {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) + + # Fallback URL might not contain a pack.toml file, if that happens - fail (for now) + # FIXME: there seems to be a lack of any kind of encoders besides JSON. + erres = + if pinfo[:fallback] do + toml_path = Path.join(pack_dir, "pack.toml") + + unless File.exists?(toml_path) do + conn + |> put_status(:internal_server_error) + |> text("No pack.toml in falblack source") + end + end + + if not is_nil(erres), do: erres, else: conn |> text("ok") + else + conn + |> put_status(:internal_server_error) + |> text("SHA256 for the pack doesn't match the one sent by the server") + end + + {:error, e} -> + conn |> put_status(:internal_server_error) |> text(e) + end + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index b9b85fd67..514446fb3 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -207,6 +207,28 @@ defmodule Pleroma.Web.Router do get("/moderation_log", AdminAPIController, :list_log) end + scope "/api/pleroma/emoji", Pleroma.Web.EmojiAPI do + scope [] do + pipe_through([:admin_api, :oauth_write]) + + post("/reload", EmojiAPIController, :reload) + end + + scope "/packs" do + # Modifying packs + pipe_through([:admin_api, :oauth_write]) + + post("/download_from", EmojiAPIController, :download_from) + end + + scope "/packs" do + # Pack info / downloading + get("/list", EmojiAPIController, :list_packs) + get("/download_shared/:name", EmojiAPIController, :download_shared) + get("/sha_of_shared/:name", EmojiAPIController, :sha_of_shared) + end + end + scope "/", Pleroma.Web.TwitterAPI do pipe_through(:pleroma_html) -- cgit v1.2.3 From 54b8e683bce13cf67f2674ea9f56b30604b28358 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 11 Aug 2019 22:32:15 +0300 Subject: Swap TOML for YAML to get YAML generation for packs from fallbacks If fallback url doesn't have a pack.yml file, one from the source will be used --- lib/pleroma/emoji.ex | 8 ++--- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 38 ++++++++++------------- lib/pleroma/web/router.ex | 1 - 3 files changed, 21 insertions(+), 26 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index ede734a53..2a9f5f804 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -143,12 +143,12 @@ defmodule Pleroma.Emoji do defp load_pack(pack_dir, emoji_groups) do pack_name = Path.basename(pack_dir) - pack_toml = Path.join(pack_dir, "pack.toml") + pack_yaml = Path.join(pack_dir, "pack.yml") - if File.exists?(pack_toml) do - toml = Toml.decode_file!(pack_toml) + if File.exists?(pack_yaml) do + yaml = RelaxYaml.Decoder.read_from_file(pack_yaml) - toml["files"] + yaml["files"] |> Enum.map(fn {name, rel_file} -> filename = Path.join("/emoji/#{pack_name}", rel_file) {name, filename, pack_name} diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 49d671518..7ef9b543d 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -22,14 +22,14 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do results |> Enum.filter(fn file -> dir_path = Path.join(@emoji_dir_path, file) - # Filter to only use the pack.toml packs - File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.toml")) + # Filter to only use the pack.yml packs + File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.yml")) end) |> Enum.map(fn pack_name -> pack_path = Path.join(@emoji_dir_path, pack_name) - pack_file = Path.join(pack_path, "pack.toml") + pack_file = Path.join(pack_path, "pack.yml") - {pack_name, Toml.decode_file!(pack_file)} + {pack_name, RelaxYaml.Decoder.read_from_file(pack_file)} end) # Transform into a map of pack-name => pack-data # Check if all the files are in place and can be sent @@ -62,7 +62,7 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do defp make_archive(name, pack, pack_dir) do files = - ['pack.toml'] ++ + ['pack.yml'] ++ (pack["files"] |> Enum.map(fn {_, path} -> to_charlist(path) end)) {:ok, {_, zip_result}} = :zip.zip('#{name}.zip', files, [:memory, cwd: to_charlist(pack_dir)]) @@ -72,10 +72,10 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do def download_shared(conn, %{"name" => name}) do pack_dir = Path.join(@emoji_dir_path, name) - pack_toml = Path.join(pack_dir, "pack.toml") + pack_yaml = Path.join(pack_dir, "pack.yml") - if File.exists?(pack_toml) do - pack = Toml.decode_file!(pack_toml) + if File.exists?(pack_yaml) do + pack = RelaxYaml.Decoder.read_from_file(pack_yaml) if can_download?(pack, pack_dir) do zip_result = make_archive(name, pack, pack_dir) @@ -139,25 +139,21 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do File.mkdir_p!(pack_dir) files = - ['pack.toml'] ++ + ['pack.yml'] ++ (pfiles |> Enum.map(fn {_, path} -> to_charlist(path) end)) {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) - # Fallback URL might not contain a pack.toml file, if that happens - fail (for now) - # FIXME: there seems to be a lack of any kind of encoders besides JSON. - erres = - if pinfo[:fallback] do - toml_path = Path.join(pack_dir, "pack.toml") - - unless File.exists?(toml_path) do - conn - |> put_status(:internal_server_error) - |> text("No pack.toml in falblack source") - end + # Fallback URL might not contain a pack.yml file. Put on we have if there's none + if pinfo[:fallback] do + yaml_path = Path.join(pack_dir, "pack.yml") + + unless File.exists?(yaml_path) do + File.write!(yaml_path, RelaxYaml.Encoder.encode(full_pack, [])) end + end - if not is_nil(erres), do: erres, else: conn |> text("ok") + conn |> text("ok") else conn |> put_status(:internal_server_error) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 514446fb3..1c781d750 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -225,7 +225,6 @@ defmodule Pleroma.Web.Router do # Pack info / downloading get("/list", EmojiAPIController, :list_packs) get("/download_shared/:name", EmojiAPIController, :download_shared) - get("/sha_of_shared/:name", EmojiAPIController, :sha_of_shared) end end -- cgit v1.2.3 From 7fb7dd9e0e0135af467477a66692990bdaecdbe9 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 11 Aug 2019 23:24:23 +0300 Subject: Only find SHA256 for packs that are shared --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 7ef9b543d..915059783 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -36,13 +36,19 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do |> Enum.map(fn {name, pack} -> pack_path = Path.join(@emoji_dir_path, name) - archive_for_sha = make_archive(name, pack, pack_path) - archive_sha = :crypto.hash(:sha256, archive_for_sha) |> Base.encode16() - - {name, - pack - |> put_in(["pack", "can-download"], can_download?(pack, pack_path)) - |> put_in(["pack", "download-sha256"], archive_sha)} + if can_download?(pack, pack_path) do + archive_for_sha = make_archive(name, pack, pack_path) + archive_sha = :crypto.hash(:sha256, archive_for_sha) |> Base.encode16() + + {name, + pack + |> put_in(["pack", "can-download"], true) + |> put_in(["pack", "download-sha256"], archive_sha)} + else + {name, + pack + |> put_in(["pack", "can-download"], false)} + end end) |> Enum.into(%{}) end -- cgit v1.2.3 From ee620ecbf11398277551ef603355a56a53690461 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 12 Aug 2019 13:13:01 +0300 Subject: Add caching for emoji pack sharing --- lib/pleroma/application.ex | 6 +++- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 42 ++++++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index dabce771d..a339e2c48 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -102,10 +102,14 @@ defmodule Pleroma.Application do build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000), build_cachex("scrubber", limit: 2500), build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500), - build_cachex("web_resp", limit: 2500) + build_cachex("web_resp", limit: 2500), + build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10) ] end + defp emoji_packs_expiration, + do: expiration(default: :timer.seconds(5 * 60), interval: :timer.seconds(60)) + defp idempotency_expiration, do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60)) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 915059783..8219eaaa1 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -1,6 +1,8 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do use Pleroma.Web, :controller + require Logger + def reload(conn, _params) do Pleroma.Emoji.reload() @@ -12,6 +14,8 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do "emoji" ) + @cache_seconds_per_file Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file]) + def list_packs(conn, _params) do pack_infos = case File.ls(@emoji_dir_path) do @@ -66,13 +70,49 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do end) end - defp make_archive(name, pack, pack_dir) do + defp create_archive_and_cache(name, pack, pack_dir, md5) do files = ['pack.yml'] ++ (pack["files"] |> Enum.map(fn {_, path} -> to_charlist(path) end)) {:ok, {_, zip_result}} = :zip.zip('#{name}.zip', files, [:memory, cwd: to_charlist(pack_dir)]) + cache_ms = :timer.seconds(@cache_seconds_per_file * Enum.count(files)) + + Cachex.put!( + :emoji_packs_cache, + name, + # if pack.yml MD5 changes, the cache is not valid anymore + %{pack_yml_md5: md5, pack_data: zip_result}, + # Add a minute to cache time for every file in the pack + ttl: cache_ms + ) + + Logger.debug("Create an archive for the '#{name}' shared emoji pack, \ +keeping it in cache for #{div(cache_ms, 1000)}s") + + zip_result + end + + defp make_archive(name, pack, pack_dir) do + # Having a different pack.yml md5 invalidates cache + pack_yml_md5 = :crypto.hash(:md5, File.read!(Path.join(pack_dir, "pack.yml"))) + + maybe_cached_pack = Cachex.get!(:emoji_packs_cache, name) + + zip_result = + if is_nil(maybe_cached_pack) do + create_archive_and_cache(name, pack, pack_dir, pack_yml_md5) + else + if maybe_cached_pack[:pack_yml_md5] == pack_yml_md5 do + Logger.debug("Using cache for the '#{name}' shared emoji pack") + + maybe_cached_pack[:pack_data] + else + create_archive_and_cache(name, pack, pack_dir, pack_yml_md5) + end + end + zip_result end -- cgit v1.2.3 From 7a0c755d0a69157868e245b35b48ed07a7dfd3c7 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 12 Aug 2019 16:43:28 +0300 Subject: Send ok for emoji reloading as text, not as json --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 8219eaaa1..72daccc8c 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do def reload(conn, _params) do Pleroma.Emoji.reload() - conn |> json("ok") + conn |> text("ok") end @emoji_dir_path Path.join( -- cgit v1.2.3 From 3a8669b48771ac4203b6abf2a372c6960d36345a Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 12 Aug 2019 17:35:25 +0300 Subject: Fix responses for emoji pack controlller --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 72daccc8c..f2b1e8a8d 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -88,7 +88,7 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do ttl: cache_ms ) - Logger.debug("Create an archive for the '#{name}' shared emoji pack, \ + Logger.debug("Create an archive for the '#{name}' emoji pack, \ keeping it in cache for #{div(cache_ms, 1000)}s") zip_result @@ -132,14 +132,14 @@ keeping it in cache for #{div(cache_ms, 1000)}s") {:error, conn |> put_status(:forbidden) - |> json("Pack #{name} cannot be downloaded from this instance, either pack sharing\ + |> text("Pack #{name} cannot be downloaded from this instance, either pack sharing\ was disabled for this pack or some files are missing")} end else {:error, conn |> put_status(:not_found) - |> json("Pack #{name} does not exist")} + |> text("Pack #{name} does not exist")} end end @@ -169,7 +169,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") }} true -> - {:error, "The pack was not set as shared and the is no fallback url to download from"} + {:error, "The pack was not set as shared and there is no fallback src to download from"} end case pack_info_res do -- cgit v1.2.3 From 2d4b8f3d20c4dbf60e52e95e77f2e77766974402 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 12 Aug 2019 18:03:59 +0300 Subject: Add an endpoint for deleting emoji packs --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 12 ++++++++++++ lib/pleroma/web/router.ex | 1 + 2 files changed, 13 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index f2b1e8a8d..49d970277 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -210,4 +210,16 @@ keeping it in cache for #{div(cache_ms, 1000)}s") conn |> put_status(:internal_server_error) |> text(e) end end + + def delete(conn, %{"name" => name}) do + pack_dir = Path.join(@emoji_dir_path, name) + + case File.rm_rf(pack_dir) do + {:ok, _} -> + conn |> text("ok") + + {:error, _} -> + conn |> put_status(:internal_server_error) |> text("Couldn't delete the pack #{name}") + end + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 1c781d750..4df0ca3c3 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -218,6 +218,7 @@ defmodule Pleroma.Web.Router do # Modifying packs pipe_through([:admin_api, :oauth_write]) + delete("/delete/:name", EmojiAPIController, :delete) post("/download_from", EmojiAPIController, :download_from) end -- cgit v1.2.3 From b0ecd412f5c499773cdc462c50d6c8104a819550 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 12 Aug 2019 18:28:05 +0300 Subject: Clean out old emojis on reload --- lib/pleroma/emoji.ex | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 2a9f5f804..f56b26da2 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -122,6 +122,9 @@ defmodule Pleroma.Emoji do fn pack -> load_pack(Path.join(emoji_dir_path, pack), emoji_groups) end ) + # Clear out old emojis + :ets.delete_all_objects(@ets) + true = :ets.insert(@ets, emojis) end -- cgit v1.2.3 From 2a94eca096f67a908410ffdd82f5bace8a3df88c Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 15 Aug 2019 11:39:39 +0300 Subject: Change YAML to JSON --- lib/pleroma/emoji.ex | 8 ++--- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 40 +++++++++++------------ 2 files changed, 24 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index f56b26da2..170a7d098 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -146,12 +146,12 @@ defmodule Pleroma.Emoji do defp load_pack(pack_dir, emoji_groups) do pack_name = Path.basename(pack_dir) - pack_yaml = Path.join(pack_dir, "pack.yml") + pack_file = Path.join(pack_dir, "pack.json") - if File.exists?(pack_yaml) do - yaml = RelaxYaml.Decoder.read_from_file(pack_yaml) + if File.exists?(pack_file) do + contents = Jason.decode!(File.read!(pack_file)) - yaml["files"] + contents["files"] |> Enum.map(fn {name, rel_file} -> filename = Path.join("/emoji/#{pack_name}", rel_file) {name, filename, pack_name} diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 49d970277..aedc70372 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -26,14 +26,14 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do results |> Enum.filter(fn file -> dir_path = Path.join(@emoji_dir_path, file) - # Filter to only use the pack.yml packs - File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.yml")) + # Filter to only use the pack.json packs + File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.json")) end) |> Enum.map(fn pack_name -> pack_path = Path.join(@emoji_dir_path, pack_name) - pack_file = Path.join(pack_path, "pack.yml") + pack_file = Path.join(pack_path, "pack.json") - {pack_name, RelaxYaml.Decoder.read_from_file(pack_file)} + {pack_name, Jason.decode!(File.read!(pack_file))} end) # Transform into a map of pack-name => pack-data # Check if all the files are in place and can be sent @@ -72,7 +72,7 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do defp create_archive_and_cache(name, pack, pack_dir, md5) do files = - ['pack.yml'] ++ + ['pack.json'] ++ (pack["files"] |> Enum.map(fn {_, path} -> to_charlist(path) end)) {:ok, {_, zip_result}} = :zip.zip('#{name}.zip', files, [:memory, cwd: to_charlist(pack_dir)]) @@ -82,8 +82,8 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do Cachex.put!( :emoji_packs_cache, name, - # if pack.yml MD5 changes, the cache is not valid anymore - %{pack_yml_md5: md5, pack_data: zip_result}, + # if pack.json MD5 changes, the cache is not valid anymore + %{pack_json_md5: md5, pack_data: zip_result}, # Add a minute to cache time for every file in the pack ttl: cache_ms ) @@ -95,21 +95,21 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end defp make_archive(name, pack, pack_dir) do - # Having a different pack.yml md5 invalidates cache - pack_yml_md5 = :crypto.hash(:md5, File.read!(Path.join(pack_dir, "pack.yml"))) + # Having a different pack.json md5 invalidates cache + pack_file_md5 = :crypto.hash(:md5, File.read!(Path.join(pack_dir, "pack.json"))) maybe_cached_pack = Cachex.get!(:emoji_packs_cache, name) zip_result = if is_nil(maybe_cached_pack) do - create_archive_and_cache(name, pack, pack_dir, pack_yml_md5) + create_archive_and_cache(name, pack, pack_dir, pack_file_md5) else - if maybe_cached_pack[:pack_yml_md5] == pack_yml_md5 do + if maybe_cached_pack[:pack_file_md5] == pack_file_md5 do Logger.debug("Using cache for the '#{name}' shared emoji pack") maybe_cached_pack[:pack_data] else - create_archive_and_cache(name, pack, pack_dir, pack_yml_md5) + create_archive_and_cache(name, pack, pack_dir, pack_file_md5) end end @@ -118,10 +118,10 @@ keeping it in cache for #{div(cache_ms, 1000)}s") def download_shared(conn, %{"name" => name}) do pack_dir = Path.join(@emoji_dir_path, name) - pack_yaml = Path.join(pack_dir, "pack.yml") + pack_file = Path.join(pack_dir, "pack.json") - if File.exists?(pack_yaml) do - pack = RelaxYaml.Decoder.read_from_file(pack_yaml) + if File.exists?(pack_file) do + pack = Jason.decode!(File.read!(pack_file)) if can_download?(pack, pack_dir) do zip_result = make_archive(name, pack, pack_dir) @@ -185,17 +185,17 @@ keeping it in cache for #{div(cache_ms, 1000)}s") File.mkdir_p!(pack_dir) files = - ['pack.yml'] ++ + ['pack.json'] ++ (pfiles |> Enum.map(fn {_, path} -> to_charlist(path) end)) {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) - # Fallback URL might not contain a pack.yml file. Put on we have if there's none + # Fallback URL might not contain a pack.json file. Put on we have if there's none if pinfo[:fallback] do - yaml_path = Path.join(pack_dir, "pack.yml") + pack_file_path = Path.join(pack_dir, "pack.json") - unless File.exists?(yaml_path) do - File.write!(yaml_path, RelaxYaml.Encoder.encode(full_pack, [])) + unless File.exists?(pack_file_path) do + File.write!(pack_file_path, Jason.encode!(full_pack)) end end -- cgit v1.2.3 From b78973d27f0c9225104914c79cf93bf3589fe7cc Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 15 Aug 2019 11:46:03 +0300 Subject: fallback can't have pack.json, reflect that in code having pacj.json and sha256 in a fallback pack would cause a circular dependency of itself --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index aedc70372..3b9eab8b8 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -184,19 +184,19 @@ keeping it in cache for #{div(cache_ms, 1000)}s") pack_dir = Path.join(@emoji_dir_path, local_name) File.mkdir_p!(pack_dir) + # Fallback cannot contain a pack.json file files = - ['pack.json'] ++ + unless(pinfo[:fallback], do: ['pack.json'], else: []) ++ (pfiles |> Enum.map(fn {_, path} -> to_charlist(path) end)) {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) - # Fallback URL might not contain a pack.json file. Put on we have if there's none + # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256 + # in it to depend on itself if pinfo[:fallback] do pack_file_path = Path.join(pack_dir, "pack.json") - unless File.exists?(pack_file_path) do - File.write!(pack_file_path, Jason.encode!(full_pack)) - end + File.write!(pack_file_path, Jason.encode!(full_pack)) end conn |> text("ok") -- cgit v1.2.3 From bcc0bfd0c54784fe6a7ccd88fc083bd09dca41af Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 15 Aug 2019 19:55:58 +0300 Subject: Add an endpoint for emoji pack metadata updating --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 49 ++++++++++++++++++++++- lib/pleroma/web/router.ex | 1 + 2 files changed, 49 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 3b9eab8b8..4096ccbed 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -196,7 +196,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") if pinfo[:fallback] do pack_file_path = Path.join(pack_dir, "pack.json") - File.write!(pack_file_path, Jason.encode!(full_pack)) + File.write!(pack_file_path, Jason.encode!(full_pack, pretty: true)) end conn |> text("ok") @@ -222,4 +222,51 @@ keeping it in cache for #{div(cache_ms, 1000)}s") conn |> put_status(:internal_server_error) |> text("Couldn't delete the pack #{name}") end end + + def update_metadata(conn, %{"name" => name, "new_data" => new_data}) do + pack_dir = Path.join(@emoji_dir_path, name) + pack_file_p = Path.join(pack_dir, "pack.json") + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + new_data = + if not is_nil(new_data["fallback-src"]) and is_nil(new_data["fallback-src-sha256"]) do + pack_arch = Tesla.get!(new_data["fallback-src"]).body + + {:ok, flist} = :zip.unzip(pack_arch, [:memory]) + + # Check if all files from the pack.json are in the archive + has_all_files = + Enum.all?(full_pack["files"], fn {_, from_manifest} -> + Enum.find(flist, fn {from_archive, _} -> + to_string(from_archive) == from_manifest + end) + end) + + unless has_all_files do + {:error, + conn + |> put_status(:bad_request) + |> text("The fallback archive does not have all files specified in pack.json")} + else + fallback_sha = :crypto.hash(:sha256, pack_arch) |> Base.encode16() + + {:ok, new_data |> Map.put("fallback-src-sha256", fallback_sha)} + end + else + {:ok, new_data} + end + + case new_data do + {:ok, new_data} -> + full_pack = Map.put(full_pack, "pack", new_data) + File.write!(pack_file_p, Jason.encode!(full_pack, pretty: true)) + + # Send new data back with fallback sha filled + conn |> json(new_data) + + {:error, e} -> + e + end + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 4df0ca3c3..471d09c43 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -218,6 +218,7 @@ defmodule Pleroma.Web.Router do # Modifying packs pipe_through([:admin_api, :oauth_write]) + post("/update_metadata/:name", EmojiAPIController, :update_metadata) delete("/delete/:name", EmojiAPIController, :delete) post("/download_from", EmojiAPIController, :download_from) end -- cgit v1.2.3 From 261d92f9c2605c720e7fce8b05025e5ac452e5c9 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Fri, 16 Aug 2019 13:30:14 +0300 Subject: Update the pack fallback-src sha generation condition The old one would not regenerate sha when fallback src changed --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 4096ccbed..4873129c4 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -229,8 +229,13 @@ keeping it in cache for #{div(cache_ms, 1000)}s") full_pack = Jason.decode!(File.read!(pack_file_p)) + # The new fallback-src is in the new data and it's not the same as it was in the old data + should_update_fb_sha = + not is_nil(new_data["fallback-src"]) and + new_data["fallback-src"] != full_pack["pack"]["fallback-src"] + new_data = - if not is_nil(new_data["fallback-src"]) and is_nil(new_data["fallback-src-sha256"]) do + if should_update_fb_sha do pack_arch = Tesla.get!(new_data["fallback-src"]).body {:ok, flist} = :zip.unzip(pack_arch, [:memory]) -- cgit v1.2.3 From 9afe7258dd5ca1e5a6333a5a9f93d9ab43d4aaf4 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 18 Aug 2019 22:05:38 +0300 Subject: Implememt emoji pack file updating + write tests --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 132 +++++++++++++++++++++- lib/pleroma/web/router.ex | 3 +- 2 files changed, 133 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 4873129c4..dc3dcf1ea 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -223,7 +223,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end end - def update_metadata(conn, %{"name" => name, "new_data" => new_data}) do + def update_metadata(conn, %{"pack_name" => name, "new_data" => new_data}) do pack_dir = Path.join(@emoji_dir_path, name) pack_file_p = Path.join(pack_dir, "pack.json") @@ -274,4 +274,134 @@ keeping it in cache for #{div(cache_ms, 1000)}s") e end end + + def update_file( + conn, + %{"pack_name" => pack_name, "action" => action, "shortcode" => shortcode} = params + ) do + pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_file_p = Path.join(pack_dir, "pack.json") + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + res = + case action do + "add" -> + unless Map.has_key?(full_pack["files"], shortcode) do + with %{"file" => %Plug.Upload{filename: filename, path: upload_path}} <- params do + # If there was a file name provided with the request, use it, otherwise just use the + # uploaded file name + filename = + if Map.has_key?(params, "filename") do + params["filename"] + else + filename + end + + file_path = Path.join(pack_dir, filename) + + # If the name contains directories, create them + if String.contains?(file_path, "/") do + File.mkdir_p!(Path.dirname(file_path)) + end + + # Copy the uploaded file from the temporary directory + File.copy!(upload_path, file_path) + + updated_full_pack = put_in(full_pack, ["files", shortcode], filename) + + {:ok, updated_full_pack} + else + _ -> {:error, conn |> put_status(:bad_request) |> text("\"file\" not provided")} + end + else + {:error, + conn + |> put_status(:conflict) + |> text("An emoji with the \"#{shortcode}\" shortcode already exists")} + end + + "remove" -> + if Map.has_key?(full_pack["files"], shortcode) do + {emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) + + emoji_file_path = Path.join(pack_dir, emoji_file_path) + + # Delete the emoji file + File.rm!(emoji_file_path) + + # If the old directory has no more files, remove it + if String.contains?(emoji_file_path, "/") do + dir = Path.dirname(emoji_file_path) + + if Enum.empty?(File.ls!(dir)) do + File.rmdir!(dir) + end + end + + {:ok, updated_full_pack} + else + {:error, + conn |> put_status(:bad_request) |> text("Emoji \"#{shortcode}\" does not exist")} + end + + "update" -> + if Map.has_key?(full_pack["files"], shortcode) do + with %{"new_shortcode" => new_shortcode, "new_filename" => new_filename} <- params do + # First, remove the old shortcode, saving the old path + {old_emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) + old_emoji_file_path = Path.join(pack_dir, old_emoji_file_path) + new_emoji_file_path = Path.join(pack_dir, new_filename) + + # If the name contains directories, create them + if String.contains?(new_emoji_file_path, "/") do + File.mkdir_p!(Path.dirname(new_emoji_file_path)) + end + + # Move/Rename the old filename to a new filename + # These are probably on the same filesystem, so just rename should work + :ok = File.rename(old_emoji_file_path, new_emoji_file_path) + + # If the old directory has no more files, remove it + if String.contains?(old_emoji_file_path, "/") do + dir = Path.dirname(old_emoji_file_path) + + if Enum.empty?(File.ls!(dir)) do + File.rmdir!(dir) + end + end + + # Then, put in the new shortcode with the new path + updated_full_pack = + put_in(updated_full_pack, ["files", new_shortcode], new_filename) + + {:ok, updated_full_pack} + else + _ -> + {:error, + conn + |> put_status(:bad_request) + |> text("new_shortcode or new_file were not specified")} + end + else + {:error, + conn |> put_status(:bad_request) |> text("Emoji \"#{shortcode}\" does not exist")} + end + + _ -> + {:error, conn |> put_status(:bad_request) |> text("Unknown action: #{action}")} + end + + case res do + {:ok, updated_full_pack} -> + # Write the emoji pack file + File.write!(pack_file_p, Jason.encode!(updated_full_pack, pretty: true)) + + # Return the modified file list + conn |> json(updated_full_pack["files"]) + + {:error, e} -> + e + end + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 471d09c43..acd6f740b 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -218,7 +218,8 @@ defmodule Pleroma.Web.Router do # Modifying packs pipe_through([:admin_api, :oauth_write]) - post("/update_metadata/:name", EmojiAPIController, :update_metadata) + post("/update_file/:pack_name", EmojiAPIController, :update_file) + post("/update_metadata/:pack_name", EmojiAPIController, :update_metadata) delete("/delete/:name", EmojiAPIController, :delete) post("/download_from", EmojiAPIController, :download_from) end -- cgit v1.2.3 From 16edfef12e6781971e2056a80a0ac38dcc254b1b Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 19 Aug 2019 19:26:15 +0300 Subject: Handle empty shortcode/filename/new_shortcode/new_filename --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 78 ++++++++++++++--------- 1 file changed, 47 insertions(+), 31 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index dc3dcf1ea..fdecbb700 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -298,19 +298,27 @@ keeping it in cache for #{div(cache_ms, 1000)}s") filename end - file_path = Path.join(pack_dir, filename) + unless String.trim(shortcode) |> String.length() == 0 or + String.trim(filename) |> String.length() == 0 do + file_path = Path.join(pack_dir, filename) - # If the name contains directories, create them - if String.contains?(file_path, "/") do - File.mkdir_p!(Path.dirname(file_path)) - end + # If the name contains directories, create them + if String.contains?(file_path, "/") do + File.mkdir_p!(Path.dirname(file_path)) + end - # Copy the uploaded file from the temporary directory - File.copy!(upload_path, file_path) + # Copy the uploaded file from the temporary directory + File.copy!(upload_path, file_path) - updated_full_pack = put_in(full_pack, ["files", shortcode], filename) + updated_full_pack = put_in(full_pack, ["files", shortcode], filename) - {:ok, updated_full_pack} + {:ok, updated_full_pack} + else + {:error, + conn + |> put_status(:bad_request) + |> text("shortcode or filename cannot be empty")} + end else _ -> {:error, conn |> put_status(:bad_request) |> text("\"file\" not provided")} end @@ -348,34 +356,42 @@ keeping it in cache for #{div(cache_ms, 1000)}s") "update" -> if Map.has_key?(full_pack["files"], shortcode) do with %{"new_shortcode" => new_shortcode, "new_filename" => new_filename} <- params do - # First, remove the old shortcode, saving the old path - {old_emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) - old_emoji_file_path = Path.join(pack_dir, old_emoji_file_path) - new_emoji_file_path = Path.join(pack_dir, new_filename) - - # If the name contains directories, create them - if String.contains?(new_emoji_file_path, "/") do - File.mkdir_p!(Path.dirname(new_emoji_file_path)) - end + unless String.trim(new_shortcode) |> String.length() == 0 or + String.trim(new_filename) |> String.length() == 0 do + # First, remove the old shortcode, saving the old path + {old_emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) + old_emoji_file_path = Path.join(pack_dir, old_emoji_file_path) + new_emoji_file_path = Path.join(pack_dir, new_filename) + + # If the name contains directories, create them + if String.contains?(new_emoji_file_path, "/") do + File.mkdir_p!(Path.dirname(new_emoji_file_path)) + end - # Move/Rename the old filename to a new filename - # These are probably on the same filesystem, so just rename should work - :ok = File.rename(old_emoji_file_path, new_emoji_file_path) + # Move/Rename the old filename to a new filename + # These are probably on the same filesystem, so just rename should work + :ok = File.rename(old_emoji_file_path, new_emoji_file_path) - # If the old directory has no more files, remove it - if String.contains?(old_emoji_file_path, "/") do - dir = Path.dirname(old_emoji_file_path) + # If the old directory has no more files, remove it + if String.contains?(old_emoji_file_path, "/") do + dir = Path.dirname(old_emoji_file_path) - if Enum.empty?(File.ls!(dir)) do - File.rmdir!(dir) + if Enum.empty?(File.ls!(dir)) do + File.rmdir!(dir) + end end - end - # Then, put in the new shortcode with the new path - updated_full_pack = - put_in(updated_full_pack, ["files", new_shortcode], new_filename) + # Then, put in the new shortcode with the new path + updated_full_pack = + put_in(updated_full_pack, ["files", new_shortcode], new_filename) - {:ok, updated_full_pack} + {:ok, updated_full_pack} + else + {:error, + conn + |> put_status(:bad_request) + |> text("new_shortcode or new_filename cannot be empty")} + end else _ -> {:error, -- cgit v1.2.3 From 8dbdd5c280d15fde4712989001d4ddee1cd37cff Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 20 Aug 2019 14:52:36 +0300 Subject: Allow uploading new emojis to packs from URLs --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 57 ++++++++++++----------- 1 file changed, 31 insertions(+), 26 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index fdecbb700..87ae0e092 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -288,39 +288,44 @@ keeping it in cache for #{div(cache_ms, 1000)}s") case action do "add" -> unless Map.has_key?(full_pack["files"], shortcode) do - with %{"file" => %Plug.Upload{filename: filename, path: upload_path}} <- params do - # If there was a file name provided with the request, use it, otherwise just use the - # uploaded file name - filename = - if Map.has_key?(params, "filename") do - params["filename"] - else - filename + filename = + if Map.has_key?(params, "filename") do + params["filename"] + else + case params["file"] do + %Plug.Upload{filename: filename} -> filename + url when is_binary(url) -> Path.basename(url) end + end - unless String.trim(shortcode) |> String.length() == 0 or - String.trim(filename) |> String.length() == 0 do - file_path = Path.join(pack_dir, filename) + unless String.trim(shortcode) |> String.length() == 0 or + String.trim(filename) |> String.length() == 0 do + file_path = Path.join(pack_dir, filename) - # If the name contains directories, create them - if String.contains?(file_path, "/") do - File.mkdir_p!(Path.dirname(file_path)) - end - - # Copy the uploaded file from the temporary directory - File.copy!(upload_path, file_path) + # If the name contains directories, create them + if String.contains?(file_path, "/") do + File.mkdir_p!(Path.dirname(file_path)) + end - updated_full_pack = put_in(full_pack, ["files", shortcode], filename) + case params["file"] do + %Plug.Upload{path: upload_path} -> + # Copy the uploaded file from the temporary directory + File.copy!(upload_path, file_path) - {:ok, updated_full_pack} - else - {:error, - conn - |> put_status(:bad_request) - |> text("shortcode or filename cannot be empty")} + url when is_binary(url) -> + # Download and write the file + file_contents = Tesla.get!(url).body + File.write!(file_path, file_contents) end + + updated_full_pack = put_in(full_pack, ["files", shortcode], filename) + + {:ok, updated_full_pack} else - _ -> {:error, conn |> put_status(:bad_request) |> text("\"file\" not provided")} + {:error, + conn + |> put_status(:bad_request) + |> text("shortcode or filename cannot be empty")} end else {:error, -- cgit v1.2.3 From f5131540dc9bbf8038e6625f4524ca01b52abbbf Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 28 Aug 2019 19:29:01 +0300 Subject: Add a way to create emoji packs via an endpoint --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 21 +++++++++++++++++++++ lib/pleroma/web/router.ex | 1 + 2 files changed, 22 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 87ae0e092..0bd9cd207 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -211,6 +211,27 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end end + def create(conn, %{"name" => name}) do + pack_dir = Path.join(@emoji_dir_path, name) + + unless File.exists?(pack_dir) do + File.mkdir_p!(pack_dir) + + pack_file_p = Path.join(pack_dir, "pack.json") + + File.write!( + pack_file_p, + Jason.encode!(%{pack: %{}, files: %{}}) + ) + + conn |> text("ok") + else + conn + |> put_status(:conflict) + |> text("A pack named \"#{name}\" already exists") + end + end + def delete(conn, %{"name" => name}) do pack_dir = Path.join(@emoji_dir_path, name) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index acd6f740b..a21fefc70 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -220,6 +220,7 @@ defmodule Pleroma.Web.Router do post("/update_file/:pack_name", EmojiAPIController, :update_file) post("/update_metadata/:pack_name", EmojiAPIController, :update_metadata) + post("/create/:name", EmojiAPIController, :create) delete("/delete/:name", EmojiAPIController, :delete) post("/download_from", EmojiAPIController, :download_from) end -- cgit v1.2.3 From 13cd93a0d314238427c217ec0ab8f59f329321f5 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 1 Sep 2019 15:38:45 +0300 Subject: Use && insted of "and" for checking shared-files for packs share-files can be nil and "and" does not like that --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 0bd9cd207..f34a4e08c 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -64,7 +64,7 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do # If the pack is set as shared, check if it can be downloaded # That means that when asked, the pack can be packed and sent to the remote # Otherwise, they'd have to download it from external-src - pack["pack"]["share-files"] and + pack["pack"]["share-files"] && Enum.all?(pack["files"], fn {_, path} -> File.exists?(Path.join(pack_path, path)) end) -- cgit v1.2.3 From 9eb2ee4df0478daec1172eec2289868105b72756 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 10 Sep 2019 21:16:30 +0300 Subject: Allow importing old (emoji.txt / plain) packs from the filesystem --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 66 +++++++++++++++++++++++ lib/pleroma/web/router.ex | 2 + 2 files changed, 68 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index f34a4e08c..dffb91b0f 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -446,4 +446,70 @@ keeping it in cache for #{div(cache_ms, 1000)}s") e end end + + def import_from_fs(conn, _params) do + case File.ls(@emoji_dir_path) do + {:error, _} -> + conn + |> put_status(:internal_server_error) + |> text("Error accessing emoji pack directory") + + {:ok, results} -> + imported_pack_names = + results + |> Enum.filter(fn file -> + dir_path = Path.join(@emoji_dir_path, file) + # Find the directories that do NOT have pack.json + File.dir?(dir_path) and not File.exists?(Path.join(dir_path, "pack.json")) + end) + |> Enum.map(fn dir -> + dir_path = Path.join(@emoji_dir_path, dir) + emoji_txt_path = Path.join(dir_path, "emoji.txt") + + files_for_pack = + if File.exists?(emoji_txt_path) do + # There's an emoji.txt file, it's likely from a pack installed by the pack manager. + # Make a pack.json file from the contents of that emoji.txt fileh + + # FIXME: Copy-pasted from Pleroma.Emoji/load_from_file_stream/2 + + # Create a map of shortcodes to filenames from emoji.txt + + File.read!(emoji_txt_path) + |> String.split("\n") + |> Enum.map(&String.trim/1) + |> Enum.map(fn line -> + case String.split(line, ~r/,\s*/) do + # This matches both strings with and without tags and we don't care about tags here + [name, file | _] -> + {name, file} + + _ -> + nil + end + end) + |> Enum.filter(fn x -> not is_nil(x) end) + |> Enum.into(%{}) + else + # If there's no emoji.txt, assume all files that are of certain extensions from the config + # are emojis and import them all + Pleroma.Emoji.make_shortcode_to_file_map( + dir_path, + Pleroma.Config.get!([:emoji, :pack_extensions]) + ) + end + + pack_json_contents = Jason.encode!(%{pack: %{}, files: files_for_pack}) + + File.write!( + Path.join(dir_path, "pack.json"), + pack_json_contents + ) + + dir + end) + + conn |> json(imported_pack_names) + end + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index a21fefc70..1252048f0 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -218,6 +218,8 @@ defmodule Pleroma.Web.Router do # Modifying packs pipe_through([:admin_api, :oauth_write]) + post("/import_from_fs", EmojiAPIController, :import_from_fs) + post("/update_file/:pack_name", EmojiAPIController, :update_file) post("/update_metadata/:pack_name", EmojiAPIController, :update_metadata) post("/create/:name", EmojiAPIController, :create) -- cgit v1.2.3 From 87057101b0e14eb51ff9367dfe9c5522ea933161 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 10 Sep 2019 21:34:57 +0300 Subject: Add documentation for the emoji api endpoints --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 52 +++++++++++++++++++++++ 1 file changed, 52 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index dffb91b0f..dc676b00f 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -16,6 +16,12 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do @cache_seconds_per_file Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file]) + @doc """ + Lists the packs available on the instance as JSON. + + The information is public and does not require authentification. The format is + a map of "pack directory name" to pack.json contents. + """ def list_packs(conn, _params) do pack_infos = case File.ls(@emoji_dir_path) do @@ -116,6 +122,10 @@ keeping it in cache for #{div(cache_ms, 1000)}s") zip_result end + @doc """ + An endpoint for other instances (via admin UI) or users (via browser) + to download packs that the instance shares. + """ def download_shared(conn, %{"name" => name}) do pack_dir = Path.join(@emoji_dir_path, name) pack_file = Path.join(pack_dir, "pack.json") @@ -143,6 +153,13 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end end + @doc """ + An admin endpoint to request downloading a pack named `pack_name` from the instance + `instance_address`. + + If the requested instance's admin chose to share the pack, it will be downloaded + from that instance, otherwise it will be downloaded from the fallback source, if there is one. + """ def download_from(conn, %{"instance_address" => address, "pack_name" => name} = data) do list_uri = "#{address}/api/pleroma/emoji/packs/list" @@ -211,6 +228,9 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end end + @doc """ + Creates an empty pack named `name` which then can be updated via the admin UI. + """ def create(conn, %{"name" => name}) do pack_dir = Path.join(@emoji_dir_path, name) @@ -232,6 +252,9 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end end + @doc """ + Deletes the pack `name` and all it's files. + """ def delete(conn, %{"name" => name}) do pack_dir = Path.join(@emoji_dir_path, name) @@ -244,6 +267,11 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end end + @doc """ + An endpoint to update `pack_names`'s metadata. + + `new_data` is the new metadata for the pack, that will replace the old metadata. + """ def update_metadata(conn, %{"pack_name" => name, "new_data" => new_data}) do pack_dir = Path.join(@emoji_dir_path, name) pack_file_p = Path.join(pack_dir, "pack.json") @@ -296,6 +324,20 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end end + @doc """ + Updates a file in a pack. + + Updating can mean three things: + + - `add` adds an emoji named `shortcode` to the pack `pack_name`, + that means that the emoji file needs to be uploaded with the request + (thus requiring it to be a multipart request) and be named `file`. + There can also be an optional `filename` that will be the new emoji file name + (if it's not there, the name will be taken from the uploaded file). + - `update` changes emoji shortcode (from `shortcode` to `new_shortcode` or moves the file + (from the current filename to `new_filename`) + - `remove` removes the emoji named `shortcode` and it's associated file + """ def update_file( conn, %{"pack_name" => pack_name, "action" => action, "shortcode" => shortcode} = params @@ -447,6 +489,16 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end end + @doc """ + Imports emoji from the filesystem. + + Importing means checking all the directories in the + `$instance_static/emoji/` for directories which do not have + `pack.json`. If one has an emoji.txt file, that file will be used + to create a `pack.json` file with it's contents. If the directory has + neither, all the files with specific configured extenstions will be + assumed to be emojis and stored in the new `pack.json` file. + """ def import_from_fs(conn, _params) do case File.ls(@emoji_dir_path) do {:error, _} -> -- cgit v1.2.3 From f6d4acc87181c94fa202ff5673f741ae9cb45b14 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 10 Sep 2019 22:09:20 +0300 Subject: Fix credo warnings --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index dc676b00f..cbd237519 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -532,7 +532,8 @@ keeping it in cache for #{div(cache_ms, 1000)}s") |> Enum.map(&String.trim/1) |> Enum.map(fn line -> case String.split(line, ~r/,\s*/) do - # This matches both strings with and without tags and we don't care about tags here + # This matches both strings with and without tags + # and we don't care about tags here [name, file | _] -> {name, file} @@ -543,8 +544,8 @@ keeping it in cache for #{div(cache_ms, 1000)}s") |> Enum.filter(fn x -> not is_nil(x) end) |> Enum.into(%{}) else - # If there's no emoji.txt, assume all files that are of certain extensions from the config - # are emojis and import them all + # If there's no emoji.txt, assume all files + # that are of certain extensions from the config are emojis and import them all Pleroma.Emoji.make_shortcode_to_file_map( dir_path, Pleroma.Config.get!([:emoji, :pack_extensions]) -- cgit v1.2.3 From 163082de6f789044b4fcb0c69f5b4cfd89731903 Mon Sep 17 00:00:00 2001 From: vaartis Date: Wed, 11 Sep 2019 09:07:19 +0000 Subject: Apply suggestion to lib/pleroma/web/emoji_api/emoji_api_controller.ex --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index cbd237519..499802fa5 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -104,22 +104,14 @@ keeping it in cache for #{div(cache_ms, 1000)}s") # Having a different pack.json md5 invalidates cache pack_file_md5 = :crypto.hash(:md5, File.read!(Path.join(pack_dir, "pack.json"))) - maybe_cached_pack = Cachex.get!(:emoji_packs_cache, name) + case Cachex.get!(:emoji_packs_cache, name) do + %{pack_file_md5: ^pack_file_md5, pack_data: zip_result} -> + Logger.debug("Using cache for the '#{name}' shared emoji pack") + zip_result - zip_result = - if is_nil(maybe_cached_pack) do + _ -> create_archive_and_cache(name, pack, pack_dir, pack_file_md5) - else - if maybe_cached_pack[:pack_file_md5] == pack_file_md5 do - Logger.debug("Using cache for the '#{name}' shared emoji pack") - - maybe_cached_pack[:pack_data] - else - create_archive_and_cache(name, pack, pack_dir, pack_file_md5) - end - end - - zip_result + end end @doc """ -- cgit v1.2.3 From c049c32270b8f70ae679e739730a3f63cdbd7d95 Mon Sep 17 00:00:00 2001 From: vaartis Date: Wed, 11 Sep 2019 09:12:22 +0000 Subject: Fixed a typo in create_archive_and_cache --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 499802fa5..51620a3eb 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -94,7 +94,7 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do ttl: cache_ms ) - Logger.debug("Create an archive for the '#{name}' emoji pack, \ + Logger.debug("Created an archive for the '#{name}' emoji pack, \ keeping it in cache for #{div(cache_ms, 1000)}s") zip_result -- cgit v1.2.3 From f251225caeede08869b472886337afea0cd47d51 Mon Sep 17 00:00:00 2001 From: vaartis Date: Wed, 11 Sep 2019 15:32:54 +0000 Subject: Apply suggestions to emoji_api_controller.ex --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 201 ++++++++++------------ 1 file changed, 95 insertions(+), 106 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 51620a3eb..0c3da6740 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -153,31 +153,32 @@ keeping it in cache for #{div(cache_ms, 1000)}s") from that instance, otherwise it will be downloaded from the fallback source, if there is one. """ def download_from(conn, %{"instance_address" => address, "pack_name" => name} = data) do - list_uri = "#{address}/api/pleroma/emoji/packs/list" - - list = Tesla.get!(list_uri).body |> Jason.decode!() - full_pack = list[name] + full_pack = + "#{address}/api/pleroma/emoji/packs/list" + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> Map.get(name) pfiles = full_pack["files"] - pack = full_pack["pack"] pack_info_res = - cond do - pack["share-files"] && pack["can-download"] -> + case full_pack["pack"] do + %{"share-files" => true, "can-download" => true, "download-sha256" => sha} -> {:ok, %{ - sha: pack["download-sha256"], + sha: sha, uri: "#{address}/api/pleroma/emoji/packs/download_shared/#{name}" }} - pack["fallback-src"] -> + %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) -> {:ok, %{ - sha: pack["fallback-src-sha256"], - uri: pack["fallback-src"], + sha: sha, + uri: src, fallback: true }} - true -> + _ -> {:error, "The pack was not set as shared and there is no fallback src to download from"} end @@ -194,9 +195,9 @@ keeping it in cache for #{div(cache_ms, 1000)}s") File.mkdir_p!(pack_dir) # Fallback cannot contain a pack.json file - files = - unless(pinfo[:fallback], do: ['pack.json'], else: []) ++ - (pfiles |> Enum.map(fn {_, path} -> to_charlist(path) end)) + files = Enum.map(full_pack["files"], fn {_, path} -> to_charlist(path) end) + # Fallback cannot contain a pack.json file + files = if pinfo[:fallback], do: files, else: ['pack.json'] ++ files {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) @@ -226,7 +227,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") def create(conn, %{"name" => name}) do pack_dir = Path.join(@emoji_dir_path, name) - unless File.exists?(pack_dir) do + if not File.exists?(pack_dir) do File.mkdir_p!(pack_dir) pack_file_p = Path.join(pack_dir, "pack.json") @@ -265,8 +266,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") `new_data` is the new metadata for the pack, that will replace the old metadata. """ def update_metadata(conn, %{"pack_name" => name, "new_data" => new_data}) do - pack_dir = Path.join(@emoji_dir_path, name) - pack_file_p = Path.join(pack_dir, "pack.json") + pack_file_p = Path.join([@emoji_dir_path, name, "pack.json"]) full_pack = Jason.decode!(File.read!(pack_file_p)) @@ -275,45 +275,40 @@ keeping it in cache for #{div(cache_ms, 1000)}s") not is_nil(new_data["fallback-src"]) and new_data["fallback-src"] != full_pack["pack"]["fallback-src"] - new_data = - if should_update_fb_sha do - pack_arch = Tesla.get!(new_data["fallback-src"]).body + with {_, true} <- {:should_update?, should_update_fb_sha}, + %{body: pack_arch} <- Tesla.get!(new_data["fallback-src"]), + {:ok, flist} <- :zip.unzip(pack_arch, [:memory]), + {_, true} <- {:has_all_files?, has_all_files?(full_pack, flist)} do + fallback_sha = :crypto.hash(:sha256, pack_arch) |> Base.encode16() - {:ok, flist} = :zip.unzip(pack_arch, [:memory]) - - # Check if all files from the pack.json are in the archive - has_all_files = - Enum.all?(full_pack["files"], fn {_, from_manifest} -> - Enum.find(flist, fn {from_archive, _} -> - to_string(from_archive) == from_manifest - end) - end) - - unless has_all_files do - {:error, - conn - |> put_status(:bad_request) - |> text("The fallback archive does not have all files specified in pack.json")} - else - fallback_sha = :crypto.hash(:sha256, pack_arch) |> Base.encode16() + new_data = Map.put(new_data, "fallback-src-sha256", fallback_sha) + update_metadata_and_send(conn, full_pack, new_data, pack_file_p) + else + {:should_update?, _} -> + update_metadata_and_send(conn, full_pack, new_data, pack_file_p) - {:ok, new_data |> Map.put("fallback-src-sha256", fallback_sha)} - end - else - {:ok, new_data} - end + {:has_all_files?, _} -> + conn + |> put_status(:bad_request) + |> text("The fallback archive does not have all files specified in pack.json") + end + end - case new_data do - {:ok, new_data} -> - full_pack = Map.put(full_pack, "pack", new_data) - File.write!(pack_file_p, Jason.encode!(full_pack, pretty: true)) + # Check if all files from the pack.json are in the archive + defp has_all_files?(%{"files" => files}, flist) do + Enum.all?(files, fn {_, from_manifest} -> + Enum.find(flist, fn {from_archive, _} -> + to_string(from_archive) == from_manifest + end) + end) + end - # Send new data back with fallback sha filled - conn |> json(new_data) + defp update_metadata_and_send(conn, full_pack, new_data, pack_file_p) do + full_pack = Map.put(full_pack, "pack", new_data) + File.write!(pack_file_p, Jason.encode!(full_pack, pretty: true)) - {:error, e} -> - e - end + # Send new data back with fallback sha filled + json(conn, new_data) end @doc """ @@ -492,69 +487,63 @@ keeping it in cache for #{div(cache_ms, 1000)}s") assumed to be emojis and stored in the new `pack.json` file. """ def import_from_fs(conn, _params) do - case File.ls(@emoji_dir_path) do + with {:ok, results} <- File.ls(@emoji_dir_path) do + imported_pack_names = + results + |> Enum.filter(fn file -> + dir_path = Path.join(@emoji_dir_path, file) + # Find the directories that do NOT have pack.json + File.dir?(dir_path) and not File.exists?(Path.join(dir_path, "pack.json")) + end) + |> Enum.map(&write_pack_json_contents/1) + + json(conn, imported_pack_names) + else {:error, _} -> conn |> put_status(:internal_server_error) |> text("Error accessing emoji pack directory") + end + end - {:ok, results} -> - imported_pack_names = - results - |> Enum.filter(fn file -> - dir_path = Path.join(@emoji_dir_path, file) - # Find the directories that do NOT have pack.json - File.dir?(dir_path) and not File.exists?(Path.join(dir_path, "pack.json")) - end) - |> Enum.map(fn dir -> - dir_path = Path.join(@emoji_dir_path, dir) - emoji_txt_path = Path.join(dir_path, "emoji.txt") - - files_for_pack = - if File.exists?(emoji_txt_path) do - # There's an emoji.txt file, it's likely from a pack installed by the pack manager. - # Make a pack.json file from the contents of that emoji.txt fileh - - # FIXME: Copy-pasted from Pleroma.Emoji/load_from_file_stream/2 - - # Create a map of shortcodes to filenames from emoji.txt - - File.read!(emoji_txt_path) - |> String.split("\n") - |> Enum.map(&String.trim/1) - |> Enum.map(fn line -> - case String.split(line, ~r/,\s*/) do - # This matches both strings with and without tags - # and we don't care about tags here - [name, file | _] -> - {name, file} - - _ -> - nil - end - end) - |> Enum.filter(fn x -> not is_nil(x) end) - |> Enum.into(%{}) - else - # If there's no emoji.txt, assume all files - # that are of certain extensions from the config are emojis and import them all - Pleroma.Emoji.make_shortcode_to_file_map( - dir_path, - Pleroma.Config.get!([:emoji, :pack_extensions]) - ) - end + defp write_pack_json_contents(dir) do + dir_path = Path.join(@emoji_dir_path, dir) + emoji_txt_path = Path.join(dir_path, "emoji.txt") - pack_json_contents = Jason.encode!(%{pack: %{}, files: files_for_pack}) + files_for_pack = files_for_pack(emoji_txt_path, dir_path) + pack_json_contents = Jason.encode!(%{pack: %{}, files: files_for_pack}) - File.write!( - Path.join(dir_path, "pack.json"), - pack_json_contents - ) + File.write!(Path.join(dir_path, "pack.json"), pack_json_contents) - dir - end) + dir + end - conn |> json(imported_pack_names) + defp files_for_pack(emoji_txt_path, dir_path) do + if File.exists?(emoji_txt_path) do + # There's an emoji.txt file, it's likely from a pack installed by the pack manager. + # Make a pack.json file from the contents of that emoji.txt fileh + + # FIXME: Copy-pasted from Pleroma.Emoji/load_from_file_stream/2 + + # Create a map of shortcodes to filenames from emoji.txt + File.read!(emoji_txt_path) + |> String.split("\n") + |> Enum.map(&String.trim/1) + |> Enum.map(fn line -> + case String.split(line, ~r/,\s*/) do + # This matches both strings with and without tags + # and we don't care about tags here + [name, file | _] -> {name, file} + _ -> nil + end + end) + |> Enum.filter(fn x -> not is_nil(x) end) + |> Enum.into(%{}) + else + # If there's no emoji.txt, assume all files + # that are of certain extensions from the config are emojis and import them all + pack_extensions = Pleroma.Config.get!([:emoji, :pack_extensions]) + Pleroma.Emoji.make_shortcode_to_file_map(dir_path, pack_extensions) end end end -- cgit v1.2.3 From b8a214b0ab264a64ca287e40e99acd401810ef58 Mon Sep 17 00:00:00 2001 From: vaartis Date: Wed, 11 Sep 2019 15:48:51 +0000 Subject: Split list_packs --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 83 ++++++++++++----------- 1 file changed, 43 insertions(+), 40 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 0c3da6740..22619f4d7 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -23,47 +23,49 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do a map of "pack directory name" to pack.json contents. """ def list_packs(conn, _params) do - pack_infos = - case File.ls(@emoji_dir_path) do - {:error, _} -> - %{} - - {:ok, results} -> - results - |> Enum.filter(fn file -> - dir_path = Path.join(@emoji_dir_path, file) - # Filter to only use the pack.json packs - File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.json")) - end) - |> Enum.map(fn pack_name -> - pack_path = Path.join(@emoji_dir_path, pack_name) - pack_file = Path.join(pack_path, "pack.json") - - {pack_name, Jason.decode!(File.read!(pack_file))} - end) - # Transform into a map of pack-name => pack-data - # Check if all the files are in place and can be sent - |> Enum.map(fn {name, pack} -> - pack_path = Path.join(@emoji_dir_path, name) - - if can_download?(pack, pack_path) do - archive_for_sha = make_archive(name, pack, pack_path) - archive_sha = :crypto.hash(:sha256, archive_for_sha) |> Base.encode16() - - {name, - pack - |> put_in(["pack", "can-download"], true) - |> put_in(["pack", "download-sha256"], archive_sha)} - else - {name, - pack - |> put_in(["pack", "can-download"], false)} - end - end) - |> Enum.into(%{}) - end + with {:ok, results} <- File.ls(@emoji_dir_path) do + pack_infos = + results + |> Enum.filter(&has_pack_json?/1) + |> Enum.map(&load_pack/1) + # Check if all the files are in place and can be sent + |> Enum.map(&validate_pack/1) + # Transform into a map of pack-name => pack-data + |> Enum.into(%{}) + + json(conn, pack_infos) + end + end + + defp has_pack_json?(file) do + dir_path = Path.join(@emoji_dir_path, file) + # Filter to only use the pack.json packs + File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.json")) + end - conn |> json(pack_infos) + defp load_pack(pack_name) do + pack_path = Path.join(@emoji_dir_path, pack_name) + pack_file = Path.join(pack_path, "pack.json") + + {pack_name, Jason.decode!(File.read!(pack_file))} + end + + defp validate_pack({name, pack}) do + pack_path = Path.join(@emoji_dir_path, name) + + if can_download?(pack, pack_path) do + archive_for_sha = make_archive(name, pack, pack_path) + archive_sha = :crypto.hash(:sha256, archive_for_sha) |> Base.encode16() + + pack = + pack + |> put_in(["pack", "can-download"], true) + |> put_in(["pack", "download-sha256"], archive_sha) + + {name, pack} + else + {name, put_in(pack, ["pack", "can-download"], false)} + end end defp can_download?(pack, pack_path) do @@ -159,6 +161,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") |> Map.get(:body) |> Jason.decode!() |> Map.get(name) + pfiles = full_pack["files"] pack_info_res = -- cgit v1.2.3 From 8790365fef9d5f76b7ac1c94933e2ee218e76285 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 11 Sep 2019 18:52:21 +0300 Subject: Remove unused variable --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 22619f4d7..8ef6ae71f 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -162,8 +162,6 @@ keeping it in cache for #{div(cache_ms, 1000)}s") |> Jason.decode!() |> Map.get(name) - pfiles = full_pack["files"] - pack_info_res = case full_pack["pack"] do %{"share-files" => true, "can-download" => true, "download-sha256" => sha} -> -- cgit v1.2.3 From 8f509e6d1ee8955fc430d1f4ed7929ba0d91177c Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 11 Sep 2019 18:59:31 +0300 Subject: Use with w/ pack_info_res --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 52 ++++++++++------------- 1 file changed, 23 insertions(+), 29 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 8ef6ae71f..9e0ff0b28 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -183,42 +183,36 @@ keeping it in cache for #{div(cache_ms, 1000)}s") {:error, "The pack was not set as shared and there is no fallback src to download from"} end - case pack_info_res do - {:ok, %{sha: sha, uri: uri} = pinfo} -> - sha = Base.decode16!(sha) - emoji_archive = Tesla.get!(uri).body - - got_sha = :crypto.hash(:sha256, emoji_archive) - - if got_sha == sha do - local_name = data["as"] || name - pack_dir = Path.join(@emoji_dir_path, local_name) - File.mkdir_p!(pack_dir) - - # Fallback cannot contain a pack.json file - files = Enum.map(full_pack["files"], fn {_, path} -> to_charlist(path) end) - # Fallback cannot contain a pack.json file - files = if pinfo[:fallback], do: files, else: ['pack.json'] ++ files + with {:ok, %{sha: sha, uri: uri} = pinfo} <- pack_info_res, + %{body: emoji_archive} <- Tesla.get!(uri), + {_, true} <- {:sha, Base.decode16!(sha) == :crypto.hash(:sha256, emoji_archive)} do + local_name = data["as"] || name + pack_dir = Path.join(@emoji_dir_path, local_name) + File.mkdir_p!(pack_dir) - {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) + files = Enum.map(full_pack["files"], fn {_, path} -> to_charlist(path) end) + # Fallback cannot contain a pack.json file + files = if pinfo[:fallback], do: files, else: ['pack.json'] ++ files - # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256 - # in it to depend on itself - if pinfo[:fallback] do - pack_file_path = Path.join(pack_dir, "pack.json") + {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) - File.write!(pack_file_path, Jason.encode!(full_pack, pretty: true)) - end + # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256 + # in it to depend on itself + if pinfo[:fallback] do + pack_file_path = Path.join(pack_dir, "pack.json") - conn |> text("ok") - else - conn - |> put_status(:internal_server_error) - |> text("SHA256 for the pack doesn't match the one sent by the server") - end + File.write!(pack_file_path, Jason.encode!(full_pack, pretty: true)) + end + text(conn, "ok") + else {:error, e} -> conn |> put_status(:internal_server_error) |> text(e) + + {:sha, _} -> + conn + |> put_status(:internal_server_error) + |> text("SHA256 for the pack doesn't match the one sent by the server") end end -- cgit v1.2.3 From cb125ffaf7f744e60fc134ef6b7b847d3838922a Mon Sep 17 00:00:00 2001 From: vaartis Date: Wed, 11 Sep 2019 16:00:48 +0000 Subject: Apply suggestion to lib/pleroma/web/emoji_api/emoji_api_controller.ex --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 32 ++++++++++------------- 1 file changed, 14 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 9e0ff0b28..28eaf5ae3 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -124,26 +124,22 @@ keeping it in cache for #{div(cache_ms, 1000)}s") pack_dir = Path.join(@emoji_dir_path, name) pack_file = Path.join(pack_dir, "pack.json") - if File.exists?(pack_file) do - pack = Jason.decode!(File.read!(pack_file)) - - if can_download?(pack, pack_dir) do - zip_result = make_archive(name, pack, pack_dir) + with {_, true} <- {:exists?, File.exists?(pack_file)}, + pack = Jason.decode!(File.read!(pack_file)), + {_, true} <- {:can_download?, can_download?(pack, pack_dir)} do + zip_result = make_archive(name, pack, pack_dir) + send_download(conn, {:binary, zip_result}, filename: "#{name}.zip") + else + {:can_download?, _} -> + conn + |> put_status(:forbidden) + |> text("Pack #{name} cannot be downloaded from this instance, either pack sharing\ + was disabled for this pack or some files are missing") + {:exists?, _} -> conn - |> send_download({:binary, zip_result}, filename: "#{name}.zip") - else - {:error, - conn - |> put_status(:forbidden) - |> text("Pack #{name} cannot be downloaded from this instance, either pack sharing\ - was disabled for this pack or some files are missing")} - end - else - {:error, - conn - |> put_status(:not_found) - |> text("Pack #{name} does not exist")} + |> put_status(:not_found) + |> text("Pack #{name} does not exist") end end -- cgit v1.2.3 From f24731788ef9dcbeb29c9dc5db9270a5787caff6 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 11 Sep 2019 19:01:21 +0300 Subject: Move emoji pack list from /list to / --- lib/pleroma/web/router.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 1252048f0..17f7406fd 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -229,7 +229,7 @@ defmodule Pleroma.Web.Router do scope "/packs" do # Pack info / downloading - get("/list", EmojiAPIController, :list_packs) + get("/", EmojiAPIController, :list_packs) get("/download_shared/:name", EmojiAPIController, :download_shared) end end -- cgit v1.2.3 From 7c784128fd8016e133c59e9c5076fa2d77a9bdee Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 11 Sep 2019 19:39:47 +0300 Subject: Change emoji api responses to JSON --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 316 ++++++++++++---------- 1 file changed, 168 insertions(+), 148 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 28eaf5ae3..1c5b7c687 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do def reload(conn, _params) do Pleroma.Emoji.reload() - conn |> text("ok") + conn |> json("ok") end @emoji_dir_path Path.join( @@ -133,13 +133,15 @@ keeping it in cache for #{div(cache_ms, 1000)}s") {:can_download?, _} -> conn |> put_status(:forbidden) - |> text("Pack #{name} cannot be downloaded from this instance, either pack sharing\ - was disabled for this pack or some files are missing") + |> json(%{ + error: "Pack #{name} cannot be downloaded from this instance, either pack sharing\ + was disabled for this pack or some files are missing" + }) {:exists?, _} -> conn |> put_status(:not_found) - |> text("Pack #{name} does not exist") + |> json(%{error: "Pack #{name} does not exist"}) end end @@ -200,15 +202,15 @@ keeping it in cache for #{div(cache_ms, 1000)}s") File.write!(pack_file_path, Jason.encode!(full_pack, pretty: true)) end - text(conn, "ok") + json(conn, "ok") else {:error, e} -> - conn |> put_status(:internal_server_error) |> text(e) + conn |> put_status(:internal_server_error) |> json(%{error: e}) {:sha, _} -> conn |> put_status(:internal_server_error) - |> text("SHA256 for the pack doesn't match the one sent by the server") + |> json(%{error: "SHA256 for the pack doesn't match the one sent by the server"}) end end @@ -228,11 +230,11 @@ keeping it in cache for #{div(cache_ms, 1000)}s") Jason.encode!(%{pack: %{}, files: %{}}) ) - conn |> text("ok") + conn |> json("ok") else conn |> put_status(:conflict) - |> text("A pack named \"#{name}\" already exists") + |> json(%{error: "A pack named \"#{name}\" already exists"}) end end @@ -244,10 +246,12 @@ keeping it in cache for #{div(cache_ms, 1000)}s") case File.rm_rf(pack_dir) do {:ok, _} -> - conn |> text("ok") + conn |> json("ok") {:error, _} -> - conn |> put_status(:internal_server_error) |> text("Couldn't delete the pack #{name}") + conn + |> put_status(:internal_server_error) + |> json(%{error: "Couldn't delete the pack #{name}"}) end end @@ -281,7 +285,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") {:has_all_files?, _} -> conn |> put_status(:bad_request) - |> text("The fallback archive does not have all files specified in pack.json") + |> json(%{error: "The fallback archive does not have all files specified in pack.json"}) end end @@ -302,6 +306,25 @@ keeping it in cache for #{div(cache_ms, 1000)}s") json(conn, new_data) end + defp get_filename(%{"filename" => filename}), do: filename + + defp get_filename(%{"file" => file}) do + case file do + %Plug.Upload{filename: filename} -> filename + url when is_binary(url) -> Path.basename(url) + end + end + + defp empty?(str), do: String.trim(str) == "" + + defp update_file_and_send(conn, updated_full_pack, pack_file_p) do + # Write the emoji pack file + File.write!(pack_file_p, Jason.encode!(updated_full_pack, pretty: true)) + + # Return the modified file list + json(conn, updated_full_pack["files"]) + end + @doc """ Updates a file in a pack. @@ -316,157 +339,154 @@ keeping it in cache for #{div(cache_ms, 1000)}s") (from the current filename to `new_filename`) - `remove` removes the emoji named `shortcode` and it's associated file """ + + # Add def update_file( conn, - %{"pack_name" => pack_name, "action" => action, "shortcode" => shortcode} = params + %{"pack_name" => pack_name, "action" => "add", "shortcode" => shortcode} = params ) do pack_dir = Path.join(@emoji_dir_path, pack_name) pack_file_p = Path.join(pack_dir, "pack.json") full_pack = Jason.decode!(File.read!(pack_file_p)) - res = - case action do - "add" -> - unless Map.has_key?(full_pack["files"], shortcode) do - filename = - if Map.has_key?(params, "filename") do - params["filename"] - else - case params["file"] do - %Plug.Upload{filename: filename} -> filename - url when is_binary(url) -> Path.basename(url) - end - end - - unless String.trim(shortcode) |> String.length() == 0 or - String.trim(filename) |> String.length() == 0 do - file_path = Path.join(pack_dir, filename) - - # If the name contains directories, create them - if String.contains?(file_path, "/") do - File.mkdir_p!(Path.dirname(file_path)) - end - - case params["file"] do - %Plug.Upload{path: upload_path} -> - # Copy the uploaded file from the temporary directory - File.copy!(upload_path, file_path) - - url when is_binary(url) -> - # Download and write the file - file_contents = Tesla.get!(url).body - File.write!(file_path, file_contents) - end - - updated_full_pack = put_in(full_pack, ["files", shortcode], filename) - - {:ok, updated_full_pack} - else - {:error, - conn - |> put_status(:bad_request) - |> text("shortcode or filename cannot be empty")} - end - else - {:error, - conn - |> put_status(:conflict) - |> text("An emoji with the \"#{shortcode}\" shortcode already exists")} - end - - "remove" -> - if Map.has_key?(full_pack["files"], shortcode) do - {emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) - - emoji_file_path = Path.join(pack_dir, emoji_file_path) - - # Delete the emoji file - File.rm!(emoji_file_path) - - # If the old directory has no more files, remove it - if String.contains?(emoji_file_path, "/") do - dir = Path.dirname(emoji_file_path) - - if Enum.empty?(File.ls!(dir)) do - File.rmdir!(dir) - end - end - - {:ok, updated_full_pack} - else - {:error, - conn |> put_status(:bad_request) |> text("Emoji \"#{shortcode}\" does not exist")} - end - - "update" -> - if Map.has_key?(full_pack["files"], shortcode) do - with %{"new_shortcode" => new_shortcode, "new_filename" => new_filename} <- params do - unless String.trim(new_shortcode) |> String.length() == 0 or - String.trim(new_filename) |> String.length() == 0 do - # First, remove the old shortcode, saving the old path - {old_emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) - old_emoji_file_path = Path.join(pack_dir, old_emoji_file_path) - new_emoji_file_path = Path.join(pack_dir, new_filename) - - # If the name contains directories, create them - if String.contains?(new_emoji_file_path, "/") do - File.mkdir_p!(Path.dirname(new_emoji_file_path)) - end - - # Move/Rename the old filename to a new filename - # These are probably on the same filesystem, so just rename should work - :ok = File.rename(old_emoji_file_path, new_emoji_file_path) - - # If the old directory has no more files, remove it - if String.contains?(old_emoji_file_path, "/") do - dir = Path.dirname(old_emoji_file_path) - - if Enum.empty?(File.ls!(dir)) do - File.rmdir!(dir) - end - end - - # Then, put in the new shortcode with the new path - updated_full_pack = - put_in(updated_full_pack, ["files", new_shortcode], new_filename) - - {:ok, updated_full_pack} - else - {:error, - conn - |> put_status(:bad_request) - |> text("new_shortcode or new_filename cannot be empty")} - end - else - _ -> - {:error, - conn - |> put_status(:bad_request) - |> text("new_shortcode or new_file were not specified")} - end - else - {:error, - conn |> put_status(:bad_request) |> text("Emoji \"#{shortcode}\" does not exist")} - end + with {_, false} <- {:has_shortcode, Map.has_key?(full_pack["files"], shortcode)}, + filename <- get_filename(params), + false <- empty?(shortcode), + false <- empty?(filename) do + file_path = Path.join(pack_dir, filename) - _ -> - {:error, conn |> put_status(:bad_request) |> text("Unknown action: #{action}")} + # If the name contains directories, create them + if String.contains?(file_path, "/") do + File.mkdir_p!(Path.dirname(file_path)) end - case res do - {:ok, updated_full_pack} -> - # Write the emoji pack file - File.write!(pack_file_p, Jason.encode!(updated_full_pack, pretty: true)) + case params["file"] do + %Plug.Upload{path: upload_path} -> + # Copy the uploaded file from the temporary directory + File.copy!(upload_path, file_path) + + url when is_binary(url) -> + # Download and write the file + file_contents = Tesla.get!(url).body + File.write!(file_path, file_contents) + end - # Return the modified file list - conn |> json(updated_full_pack["files"]) + updated_full_pack = put_in(full_pack, ["files", shortcode], filename) + update_file_and_send(conn, updated_full_pack, pack_file_p) + else + {:has_shortcode, _} -> + conn + |> put_status(:conflict) + |> json(%{error: "An emoji with the \"#{shortcode}\" shortcode already exists"}) - {:error, e} -> - e + true -> + conn + |> put_status(:bad_request) + |> json(%{error: "shortcode or filename cannot be empty"}) + end + end + + # Remove + def update_file(conn, %{ + "pack_name" => pack_name, + "action" => "remove", + "shortcode" => shortcode + }) do + pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_file_p = Path.join(pack_dir, "pack.json") + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + if Map.has_key?(full_pack["files"], shortcode) do + {emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) + + emoji_file_path = Path.join(pack_dir, emoji_file_path) + + # Delete the emoji file + File.rm!(emoji_file_path) + + # If the old directory has no more files, remove it + if String.contains?(emoji_file_path, "/") do + dir = Path.dirname(emoji_file_path) + + if Enum.empty?(File.ls!(dir)) do + File.rmdir!(dir) + end + end + + update_file_and_send(conn, updated_full_pack, pack_file_p) + else + conn + |> put_status(:bad_request) + |> json(%{error: "Emoji \"#{shortcode}\" does not exist"}) end end + # Update + def update_file( + conn, + %{"pack_name" => pack_name, "action" => "update", "shortcode" => shortcode} = params + ) do + pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_file_p = Path.join(pack_dir, "pack.json") + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + with {_, true} <- {:has_shortcode, Map.has_key?(full_pack["files"], shortcode)}, + %{"new_shortcode" => new_shortcode, "new_filename" => new_filename} <- params, + false <- empty?(new_shortcode), + false <- empty?(new_filename) do + # First, remove the old shortcode, saving the old path + {old_emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) + old_emoji_file_path = Path.join(pack_dir, old_emoji_file_path) + new_emoji_file_path = Path.join(pack_dir, new_filename) + + # If the name contains directories, create them + if String.contains?(new_emoji_file_path, "/") do + File.mkdir_p!(Path.dirname(new_emoji_file_path)) + end + + # Move/Rename the old filename to a new filename + # These are probably on the same filesystem, so just rename should work + :ok = File.rename(old_emoji_file_path, new_emoji_file_path) + + # If the old directory has no more files, remove it + if String.contains?(old_emoji_file_path, "/") do + dir = Path.dirname(old_emoji_file_path) + + if Enum.empty?(File.ls!(dir)) do + File.rmdir!(dir) + end + end + + # Then, put in the new shortcode with the new path + updated_full_pack = put_in(updated_full_pack, ["files", new_shortcode], new_filename) + update_file_and_send(conn, updated_full_pack, pack_file_p) + else + {:has_shortcode, _} -> + conn + |> put_status(:bad_request) + |> json(%{error: "Emoji \"#{shortcode}\" does not exist"}) + + true -> + conn + |> put_status(:bad_request) + |> json(%{error: "new_shortcode or new_filename cannot be empty"}) + + _ -> + conn + |> put_status(:bad_request) + |> json(%{error: "new_shortcode or new_file were not specified"}) + end + end + + def update_file(conn, %{"action" => action}) do + conn + |> put_status(:bad_request) + |> json(%{error: "Unknown action: #{action}"}) + end + @doc """ Imports emoji from the filesystem. @@ -493,7 +513,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") {:error, _} -> conn |> put_status(:internal_server_error) - |> text("Error accessing emoji pack directory") + |> json(%{error: "Error accessing emoji pack directory"}) end end -- cgit v1.2.3 From 3971bf9c5f00d12a0a2048eb3676069d58a9f243 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 11 Sep 2019 21:43:16 +0300 Subject: Change :sha to :checksum --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 1c5b7c687..0d4a17c61 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -183,7 +183,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") with {:ok, %{sha: sha, uri: uri} = pinfo} <- pack_info_res, %{body: emoji_archive} <- Tesla.get!(uri), - {_, true} <- {:sha, Base.decode16!(sha) == :crypto.hash(:sha256, emoji_archive)} do + {_, true} <- {:checksum, Base.decode16!(sha) == :crypto.hash(:sha256, emoji_archive)} do local_name = data["as"] || name pack_dir = Path.join(@emoji_dir_path, local_name) File.mkdir_p!(pack_dir) @@ -207,7 +207,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") {:error, e} -> conn |> put_status(:internal_server_error) |> json(%{error: e}) - {:sha, _} -> + {:checksum, _} -> conn |> put_status(:internal_server_error) |> json(%{error: "SHA256 for the pack doesn't match the one sent by the server"}) -- cgit v1.2.3 From 6cd651a38be898456c06d8fee7fd15f1b406848c Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 11 Sep 2019 21:50:55 +0300 Subject: Make the emoji controller api more RESTy --- lib/pleroma/web/router.ex | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 17f7406fd..bae25c60a 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -220,17 +220,17 @@ defmodule Pleroma.Web.Router do post("/import_from_fs", EmojiAPIController, :import_from_fs) - post("/update_file/:pack_name", EmojiAPIController, :update_file) - post("/update_metadata/:pack_name", EmojiAPIController, :update_metadata) - post("/create/:name", EmojiAPIController, :create) - delete("/delete/:name", EmojiAPIController, :delete) + post("/:pack_name/update_file", EmojiAPIController, :update_file) + post("/:pack_name/update_metadata", EmojiAPIController, :update_metadata) + put("/:name", EmojiAPIController, :create) + delete("/:name", EmojiAPIController, :delete) post("/download_from", EmojiAPIController, :download_from) end scope "/packs" do # Pack info / downloading get("/", EmojiAPIController, :list_packs) - get("/download_shared/:name", EmojiAPIController, :download_shared) + get("/:name/download_shared/", EmojiAPIController, :download_shared) end end -- cgit v1.2.3 From 74fb6d864760ccaa18b9a20d148c590254779454 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 11 Sep 2019 22:43:00 +0300 Subject: Move EmojiAPIController from EmojiAPI to PleromaAPI --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 2 +- lib/pleroma/web/router.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 0d4a17c61..a83f8af57 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -1,4 +1,4 @@ -defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do +defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do use Pleroma.Web, :controller require Logger diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index bae25c60a..715e4ba68 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -207,7 +207,7 @@ defmodule Pleroma.Web.Router do get("/moderation_log", AdminAPIController, :list_log) end - scope "/api/pleroma/emoji", Pleroma.Web.EmojiAPI do + scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do scope [] do pipe_through([:admin_api, :oauth_write]) -- cgit v1.2.3 From 36f2275dc9f6c58163e4e07f8ace9d75e96033c7 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 11 Sep 2019 22:58:55 +0300 Subject: A feature for shareable emoji packs, use it in download_from & tests --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 111 ++++++++++++---------- lib/pleroma/web/nodeinfo/nodeinfo_controller.ex | 1 + 2 files changed, 64 insertions(+), 48 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index a83f8af57..36ca2c804 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -153,64 +153,79 @@ keeping it in cache for #{div(cache_ms, 1000)}s") from that instance, otherwise it will be downloaded from the fallback source, if there is one. """ def download_from(conn, %{"instance_address" => address, "pack_name" => name} = data) do - full_pack = - "#{address}/api/pleroma/emoji/packs/list" + shareable_packs_available = + "#{address}/nodeinfo/2.1.json" |> Tesla.get!() |> Map.get(:body) |> Jason.decode!() - |> Map.get(name) - - pack_info_res = - case full_pack["pack"] do - %{"share-files" => true, "can-download" => true, "download-sha256" => sha} -> - {:ok, - %{ - sha: sha, - uri: "#{address}/api/pleroma/emoji/packs/download_shared/#{name}" - }} - - %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) -> - {:ok, - %{ - sha: sha, - uri: src, - fallback: true - }} - - _ -> - {:error, "The pack was not set as shared and there is no fallback src to download from"} - end + |> Map.get("features") + |> Enum.member?("shareable_emoji_packs") + + if shareable_packs_available do + full_pack = + "#{address}/api/pleroma/emoji/packs/list" + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> Map.get(name) + + pack_info_res = + case full_pack["pack"] do + %{"share-files" => true, "can-download" => true, "download-sha256" => sha} -> + {:ok, + %{ + sha: sha, + uri: "#{address}/api/pleroma/emoji/packs/download_shared/#{name}" + }} + + %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) -> + {:ok, + %{ + sha: sha, + uri: src, + fallback: true + }} + + _ -> + {:error, + "The pack was not set as shared and there is no fallback src to download from"} + end - with {:ok, %{sha: sha, uri: uri} = pinfo} <- pack_info_res, - %{body: emoji_archive} <- Tesla.get!(uri), - {_, true} <- {:checksum, Base.decode16!(sha) == :crypto.hash(:sha256, emoji_archive)} do - local_name = data["as"] || name - pack_dir = Path.join(@emoji_dir_path, local_name) - File.mkdir_p!(pack_dir) + with {:ok, %{sha: sha, uri: uri} = pinfo} <- pack_info_res, + %{body: emoji_archive} <- Tesla.get!(uri), + {_, true} <- {:checksum, Base.decode16!(sha) == :crypto.hash(:sha256, emoji_archive)} do + local_name = data["as"] || name + pack_dir = Path.join(@emoji_dir_path, local_name) + File.mkdir_p!(pack_dir) - files = Enum.map(full_pack["files"], fn {_, path} -> to_charlist(path) end) - # Fallback cannot contain a pack.json file - files = if pinfo[:fallback], do: files, else: ['pack.json'] ++ files + files = Enum.map(full_pack["files"], fn {_, path} -> to_charlist(path) end) + # Fallback cannot contain a pack.json file + files = if pinfo[:fallback], do: files, else: ['pack.json'] ++ files - {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) + {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) - # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256 - # in it to depend on itself - if pinfo[:fallback] do - pack_file_path = Path.join(pack_dir, "pack.json") + # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256 + # in it to depend on itself + if pinfo[:fallback] do + pack_file_path = Path.join(pack_dir, "pack.json") - File.write!(pack_file_path, Jason.encode!(full_pack, pretty: true)) - end + File.write!(pack_file_path, Jason.encode!(full_pack, pretty: true)) + end - json(conn, "ok") - else - {:error, e} -> - conn |> put_status(:internal_server_error) |> json(%{error: e}) + json(conn, "ok") + else + {:error, e} -> + conn |> put_status(:internal_server_error) |> json(%{error: e}) - {:checksum, _} -> - conn - |> put_status(:internal_server_error) - |> json(%{error: "SHA256 for the pack doesn't match the one sent by the server"}) + {:checksum, _} -> + conn + |> put_status(:internal_server_error) + |> json(%{error: "SHA256 for the pack doesn't match the one sent by the server"}) + end + else + conn + |> put_status(:internal_server_error) + |> json(%{error: "The requested instance does not support sharing emoji packs"}) end end diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index ee14cfd6b..192984242 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -57,6 +57,7 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do "mastodon_api_streaming", "polls", "pleroma_explicit_addressing", + "shareable_emoji_packs", if Config.get([:media_proxy, :enabled]) do "media_proxy" end, -- cgit v1.2.3 From 7680aec17d6690ccf7383354572456c2118a8750 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 12 Sep 2019 00:00:28 +0300 Subject: Move emoji api to pleroma api dir --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 575 --------------------- .../web/pleroma_api/emoji_api_controller.ex | 575 +++++++++++++++++++++ 2 files changed, 575 insertions(+), 575 deletions(-) delete mode 100644 lib/pleroma/web/emoji_api/emoji_api_controller.ex create mode 100644 lib/pleroma/web/pleroma_api/emoji_api_controller.ex (limited to 'lib') diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex deleted file mode 100644 index 36ca2c804..000000000 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ /dev/null @@ -1,575 +0,0 @@ -defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do - use Pleroma.Web, :controller - - require Logger - - def reload(conn, _params) do - Pleroma.Emoji.reload() - - conn |> json("ok") - end - - @emoji_dir_path Path.join( - Pleroma.Config.get!([:instance, :static_dir]), - "emoji" - ) - - @cache_seconds_per_file Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file]) - - @doc """ - Lists the packs available on the instance as JSON. - - The information is public and does not require authentification. The format is - a map of "pack directory name" to pack.json contents. - """ - def list_packs(conn, _params) do - with {:ok, results} <- File.ls(@emoji_dir_path) do - pack_infos = - results - |> Enum.filter(&has_pack_json?/1) - |> Enum.map(&load_pack/1) - # Check if all the files are in place and can be sent - |> Enum.map(&validate_pack/1) - # Transform into a map of pack-name => pack-data - |> Enum.into(%{}) - - json(conn, pack_infos) - end - end - - defp has_pack_json?(file) do - dir_path = Path.join(@emoji_dir_path, file) - # Filter to only use the pack.json packs - File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.json")) - end - - defp load_pack(pack_name) do - pack_path = Path.join(@emoji_dir_path, pack_name) - pack_file = Path.join(pack_path, "pack.json") - - {pack_name, Jason.decode!(File.read!(pack_file))} - end - - defp validate_pack({name, pack}) do - pack_path = Path.join(@emoji_dir_path, name) - - if can_download?(pack, pack_path) do - archive_for_sha = make_archive(name, pack, pack_path) - archive_sha = :crypto.hash(:sha256, archive_for_sha) |> Base.encode16() - - pack = - pack - |> put_in(["pack", "can-download"], true) - |> put_in(["pack", "download-sha256"], archive_sha) - - {name, pack} - else - {name, put_in(pack, ["pack", "can-download"], false)} - end - end - - defp can_download?(pack, pack_path) do - # If the pack is set as shared, check if it can be downloaded - # That means that when asked, the pack can be packed and sent to the remote - # Otherwise, they'd have to download it from external-src - pack["pack"]["share-files"] && - Enum.all?(pack["files"], fn {_, path} -> - File.exists?(Path.join(pack_path, path)) - end) - end - - defp create_archive_and_cache(name, pack, pack_dir, md5) do - files = - ['pack.json'] ++ - (pack["files"] |> Enum.map(fn {_, path} -> to_charlist(path) end)) - - {:ok, {_, zip_result}} = :zip.zip('#{name}.zip', files, [:memory, cwd: to_charlist(pack_dir)]) - - cache_ms = :timer.seconds(@cache_seconds_per_file * Enum.count(files)) - - Cachex.put!( - :emoji_packs_cache, - name, - # if pack.json MD5 changes, the cache is not valid anymore - %{pack_json_md5: md5, pack_data: zip_result}, - # Add a minute to cache time for every file in the pack - ttl: cache_ms - ) - - Logger.debug("Created an archive for the '#{name}' emoji pack, \ -keeping it in cache for #{div(cache_ms, 1000)}s") - - zip_result - end - - defp make_archive(name, pack, pack_dir) do - # Having a different pack.json md5 invalidates cache - pack_file_md5 = :crypto.hash(:md5, File.read!(Path.join(pack_dir, "pack.json"))) - - case Cachex.get!(:emoji_packs_cache, name) do - %{pack_file_md5: ^pack_file_md5, pack_data: zip_result} -> - Logger.debug("Using cache for the '#{name}' shared emoji pack") - zip_result - - _ -> - create_archive_and_cache(name, pack, pack_dir, pack_file_md5) - end - end - - @doc """ - An endpoint for other instances (via admin UI) or users (via browser) - to download packs that the instance shares. - """ - def download_shared(conn, %{"name" => name}) do - pack_dir = Path.join(@emoji_dir_path, name) - pack_file = Path.join(pack_dir, "pack.json") - - with {_, true} <- {:exists?, File.exists?(pack_file)}, - pack = Jason.decode!(File.read!(pack_file)), - {_, true} <- {:can_download?, can_download?(pack, pack_dir)} do - zip_result = make_archive(name, pack, pack_dir) - send_download(conn, {:binary, zip_result}, filename: "#{name}.zip") - else - {:can_download?, _} -> - conn - |> put_status(:forbidden) - |> json(%{ - error: "Pack #{name} cannot be downloaded from this instance, either pack sharing\ - was disabled for this pack or some files are missing" - }) - - {:exists?, _} -> - conn - |> put_status(:not_found) - |> json(%{error: "Pack #{name} does not exist"}) - end - end - - @doc """ - An admin endpoint to request downloading a pack named `pack_name` from the instance - `instance_address`. - - If the requested instance's admin chose to share the pack, it will be downloaded - from that instance, otherwise it will be downloaded from the fallback source, if there is one. - """ - def download_from(conn, %{"instance_address" => address, "pack_name" => name} = data) do - shareable_packs_available = - "#{address}/nodeinfo/2.1.json" - |> Tesla.get!() - |> Map.get(:body) - |> Jason.decode!() - |> Map.get("features") - |> Enum.member?("shareable_emoji_packs") - - if shareable_packs_available do - full_pack = - "#{address}/api/pleroma/emoji/packs/list" - |> Tesla.get!() - |> Map.get(:body) - |> Jason.decode!() - |> Map.get(name) - - pack_info_res = - case full_pack["pack"] do - %{"share-files" => true, "can-download" => true, "download-sha256" => sha} -> - {:ok, - %{ - sha: sha, - uri: "#{address}/api/pleroma/emoji/packs/download_shared/#{name}" - }} - - %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) -> - {:ok, - %{ - sha: sha, - uri: src, - fallback: true - }} - - _ -> - {:error, - "The pack was not set as shared and there is no fallback src to download from"} - end - - with {:ok, %{sha: sha, uri: uri} = pinfo} <- pack_info_res, - %{body: emoji_archive} <- Tesla.get!(uri), - {_, true} <- {:checksum, Base.decode16!(sha) == :crypto.hash(:sha256, emoji_archive)} do - local_name = data["as"] || name - pack_dir = Path.join(@emoji_dir_path, local_name) - File.mkdir_p!(pack_dir) - - files = Enum.map(full_pack["files"], fn {_, path} -> to_charlist(path) end) - # Fallback cannot contain a pack.json file - files = if pinfo[:fallback], do: files, else: ['pack.json'] ++ files - - {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) - - # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256 - # in it to depend on itself - if pinfo[:fallback] do - pack_file_path = Path.join(pack_dir, "pack.json") - - File.write!(pack_file_path, Jason.encode!(full_pack, pretty: true)) - end - - json(conn, "ok") - else - {:error, e} -> - conn |> put_status(:internal_server_error) |> json(%{error: e}) - - {:checksum, _} -> - conn - |> put_status(:internal_server_error) - |> json(%{error: "SHA256 for the pack doesn't match the one sent by the server"}) - end - else - conn - |> put_status(:internal_server_error) - |> json(%{error: "The requested instance does not support sharing emoji packs"}) - end - end - - @doc """ - Creates an empty pack named `name` which then can be updated via the admin UI. - """ - def create(conn, %{"name" => name}) do - pack_dir = Path.join(@emoji_dir_path, name) - - if not File.exists?(pack_dir) do - File.mkdir_p!(pack_dir) - - pack_file_p = Path.join(pack_dir, "pack.json") - - File.write!( - pack_file_p, - Jason.encode!(%{pack: %{}, files: %{}}) - ) - - conn |> json("ok") - else - conn - |> put_status(:conflict) - |> json(%{error: "A pack named \"#{name}\" already exists"}) - end - end - - @doc """ - Deletes the pack `name` and all it's files. - """ - def delete(conn, %{"name" => name}) do - pack_dir = Path.join(@emoji_dir_path, name) - - case File.rm_rf(pack_dir) do - {:ok, _} -> - conn |> json("ok") - - {:error, _} -> - conn - |> put_status(:internal_server_error) - |> json(%{error: "Couldn't delete the pack #{name}"}) - end - end - - @doc """ - An endpoint to update `pack_names`'s metadata. - - `new_data` is the new metadata for the pack, that will replace the old metadata. - """ - def update_metadata(conn, %{"pack_name" => name, "new_data" => new_data}) do - pack_file_p = Path.join([@emoji_dir_path, name, "pack.json"]) - - full_pack = Jason.decode!(File.read!(pack_file_p)) - - # The new fallback-src is in the new data and it's not the same as it was in the old data - should_update_fb_sha = - not is_nil(new_data["fallback-src"]) and - new_data["fallback-src"] != full_pack["pack"]["fallback-src"] - - with {_, true} <- {:should_update?, should_update_fb_sha}, - %{body: pack_arch} <- Tesla.get!(new_data["fallback-src"]), - {:ok, flist} <- :zip.unzip(pack_arch, [:memory]), - {_, true} <- {:has_all_files?, has_all_files?(full_pack, flist)} do - fallback_sha = :crypto.hash(:sha256, pack_arch) |> Base.encode16() - - new_data = Map.put(new_data, "fallback-src-sha256", fallback_sha) - update_metadata_and_send(conn, full_pack, new_data, pack_file_p) - else - {:should_update?, _} -> - update_metadata_and_send(conn, full_pack, new_data, pack_file_p) - - {:has_all_files?, _} -> - conn - |> put_status(:bad_request) - |> json(%{error: "The fallback archive does not have all files specified in pack.json"}) - end - end - - # Check if all files from the pack.json are in the archive - defp has_all_files?(%{"files" => files}, flist) do - Enum.all?(files, fn {_, from_manifest} -> - Enum.find(flist, fn {from_archive, _} -> - to_string(from_archive) == from_manifest - end) - end) - end - - defp update_metadata_and_send(conn, full_pack, new_data, pack_file_p) do - full_pack = Map.put(full_pack, "pack", new_data) - File.write!(pack_file_p, Jason.encode!(full_pack, pretty: true)) - - # Send new data back with fallback sha filled - json(conn, new_data) - end - - defp get_filename(%{"filename" => filename}), do: filename - - defp get_filename(%{"file" => file}) do - case file do - %Plug.Upload{filename: filename} -> filename - url when is_binary(url) -> Path.basename(url) - end - end - - defp empty?(str), do: String.trim(str) == "" - - defp update_file_and_send(conn, updated_full_pack, pack_file_p) do - # Write the emoji pack file - File.write!(pack_file_p, Jason.encode!(updated_full_pack, pretty: true)) - - # Return the modified file list - json(conn, updated_full_pack["files"]) - end - - @doc """ - Updates a file in a pack. - - Updating can mean three things: - - - `add` adds an emoji named `shortcode` to the pack `pack_name`, - that means that the emoji file needs to be uploaded with the request - (thus requiring it to be a multipart request) and be named `file`. - There can also be an optional `filename` that will be the new emoji file name - (if it's not there, the name will be taken from the uploaded file). - - `update` changes emoji shortcode (from `shortcode` to `new_shortcode` or moves the file - (from the current filename to `new_filename`) - - `remove` removes the emoji named `shortcode` and it's associated file - """ - - # Add - def update_file( - conn, - %{"pack_name" => pack_name, "action" => "add", "shortcode" => shortcode} = params - ) do - pack_dir = Path.join(@emoji_dir_path, pack_name) - pack_file_p = Path.join(pack_dir, "pack.json") - - full_pack = Jason.decode!(File.read!(pack_file_p)) - - with {_, false} <- {:has_shortcode, Map.has_key?(full_pack["files"], shortcode)}, - filename <- get_filename(params), - false <- empty?(shortcode), - false <- empty?(filename) do - file_path = Path.join(pack_dir, filename) - - # If the name contains directories, create them - if String.contains?(file_path, "/") do - File.mkdir_p!(Path.dirname(file_path)) - end - - case params["file"] do - %Plug.Upload{path: upload_path} -> - # Copy the uploaded file from the temporary directory - File.copy!(upload_path, file_path) - - url when is_binary(url) -> - # Download and write the file - file_contents = Tesla.get!(url).body - File.write!(file_path, file_contents) - end - - updated_full_pack = put_in(full_pack, ["files", shortcode], filename) - update_file_and_send(conn, updated_full_pack, pack_file_p) - else - {:has_shortcode, _} -> - conn - |> put_status(:conflict) - |> json(%{error: "An emoji with the \"#{shortcode}\" shortcode already exists"}) - - true -> - conn - |> put_status(:bad_request) - |> json(%{error: "shortcode or filename cannot be empty"}) - end - end - - # Remove - def update_file(conn, %{ - "pack_name" => pack_name, - "action" => "remove", - "shortcode" => shortcode - }) do - pack_dir = Path.join(@emoji_dir_path, pack_name) - pack_file_p = Path.join(pack_dir, "pack.json") - - full_pack = Jason.decode!(File.read!(pack_file_p)) - - if Map.has_key?(full_pack["files"], shortcode) do - {emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) - - emoji_file_path = Path.join(pack_dir, emoji_file_path) - - # Delete the emoji file - File.rm!(emoji_file_path) - - # If the old directory has no more files, remove it - if String.contains?(emoji_file_path, "/") do - dir = Path.dirname(emoji_file_path) - - if Enum.empty?(File.ls!(dir)) do - File.rmdir!(dir) - end - end - - update_file_and_send(conn, updated_full_pack, pack_file_p) - else - conn - |> put_status(:bad_request) - |> json(%{error: "Emoji \"#{shortcode}\" does not exist"}) - end - end - - # Update - def update_file( - conn, - %{"pack_name" => pack_name, "action" => "update", "shortcode" => shortcode} = params - ) do - pack_dir = Path.join(@emoji_dir_path, pack_name) - pack_file_p = Path.join(pack_dir, "pack.json") - - full_pack = Jason.decode!(File.read!(pack_file_p)) - - with {_, true} <- {:has_shortcode, Map.has_key?(full_pack["files"], shortcode)}, - %{"new_shortcode" => new_shortcode, "new_filename" => new_filename} <- params, - false <- empty?(new_shortcode), - false <- empty?(new_filename) do - # First, remove the old shortcode, saving the old path - {old_emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) - old_emoji_file_path = Path.join(pack_dir, old_emoji_file_path) - new_emoji_file_path = Path.join(pack_dir, new_filename) - - # If the name contains directories, create them - if String.contains?(new_emoji_file_path, "/") do - File.mkdir_p!(Path.dirname(new_emoji_file_path)) - end - - # Move/Rename the old filename to a new filename - # These are probably on the same filesystem, so just rename should work - :ok = File.rename(old_emoji_file_path, new_emoji_file_path) - - # If the old directory has no more files, remove it - if String.contains?(old_emoji_file_path, "/") do - dir = Path.dirname(old_emoji_file_path) - - if Enum.empty?(File.ls!(dir)) do - File.rmdir!(dir) - end - end - - # Then, put in the new shortcode with the new path - updated_full_pack = put_in(updated_full_pack, ["files", new_shortcode], new_filename) - update_file_and_send(conn, updated_full_pack, pack_file_p) - else - {:has_shortcode, _} -> - conn - |> put_status(:bad_request) - |> json(%{error: "Emoji \"#{shortcode}\" does not exist"}) - - true -> - conn - |> put_status(:bad_request) - |> json(%{error: "new_shortcode or new_filename cannot be empty"}) - - _ -> - conn - |> put_status(:bad_request) - |> json(%{error: "new_shortcode or new_file were not specified"}) - end - end - - def update_file(conn, %{"action" => action}) do - conn - |> put_status(:bad_request) - |> json(%{error: "Unknown action: #{action}"}) - end - - @doc """ - Imports emoji from the filesystem. - - Importing means checking all the directories in the - `$instance_static/emoji/` for directories which do not have - `pack.json`. If one has an emoji.txt file, that file will be used - to create a `pack.json` file with it's contents. If the directory has - neither, all the files with specific configured extenstions will be - assumed to be emojis and stored in the new `pack.json` file. - """ - def import_from_fs(conn, _params) do - with {:ok, results} <- File.ls(@emoji_dir_path) do - imported_pack_names = - results - |> Enum.filter(fn file -> - dir_path = Path.join(@emoji_dir_path, file) - # Find the directories that do NOT have pack.json - File.dir?(dir_path) and not File.exists?(Path.join(dir_path, "pack.json")) - end) - |> Enum.map(&write_pack_json_contents/1) - - json(conn, imported_pack_names) - else - {:error, _} -> - conn - |> put_status(:internal_server_error) - |> json(%{error: "Error accessing emoji pack directory"}) - end - end - - defp write_pack_json_contents(dir) do - dir_path = Path.join(@emoji_dir_path, dir) - emoji_txt_path = Path.join(dir_path, "emoji.txt") - - files_for_pack = files_for_pack(emoji_txt_path, dir_path) - pack_json_contents = Jason.encode!(%{pack: %{}, files: files_for_pack}) - - File.write!(Path.join(dir_path, "pack.json"), pack_json_contents) - - dir - end - - defp files_for_pack(emoji_txt_path, dir_path) do - if File.exists?(emoji_txt_path) do - # There's an emoji.txt file, it's likely from a pack installed by the pack manager. - # Make a pack.json file from the contents of that emoji.txt fileh - - # FIXME: Copy-pasted from Pleroma.Emoji/load_from_file_stream/2 - - # Create a map of shortcodes to filenames from emoji.txt - File.read!(emoji_txt_path) - |> String.split("\n") - |> Enum.map(&String.trim/1) - |> Enum.map(fn line -> - case String.split(line, ~r/,\s*/) do - # This matches both strings with and without tags - # and we don't care about tags here - [name, file | _] -> {name, file} - _ -> nil - end - end) - |> Enum.filter(fn x -> not is_nil(x) end) - |> Enum.into(%{}) - else - # If there's no emoji.txt, assume all files - # that are of certain extensions from the config are emojis and import them all - pack_extensions = Pleroma.Config.get!([:emoji, :pack_extensions]) - Pleroma.Emoji.make_shortcode_to_file_map(dir_path, pack_extensions) - end - end -end diff --git a/lib/pleroma/web/pleroma_api/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/emoji_api_controller.ex new file mode 100644 index 000000000..36ca2c804 --- /dev/null +++ b/lib/pleroma/web/pleroma_api/emoji_api_controller.ex @@ -0,0 +1,575 @@ +defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do + use Pleroma.Web, :controller + + require Logger + + def reload(conn, _params) do + Pleroma.Emoji.reload() + + conn |> json("ok") + end + + @emoji_dir_path Path.join( + Pleroma.Config.get!([:instance, :static_dir]), + "emoji" + ) + + @cache_seconds_per_file Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file]) + + @doc """ + Lists the packs available on the instance as JSON. + + The information is public and does not require authentification. The format is + a map of "pack directory name" to pack.json contents. + """ + def list_packs(conn, _params) do + with {:ok, results} <- File.ls(@emoji_dir_path) do + pack_infos = + results + |> Enum.filter(&has_pack_json?/1) + |> Enum.map(&load_pack/1) + # Check if all the files are in place and can be sent + |> Enum.map(&validate_pack/1) + # Transform into a map of pack-name => pack-data + |> Enum.into(%{}) + + json(conn, pack_infos) + end + end + + defp has_pack_json?(file) do + dir_path = Path.join(@emoji_dir_path, file) + # Filter to only use the pack.json packs + File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.json")) + end + + defp load_pack(pack_name) do + pack_path = Path.join(@emoji_dir_path, pack_name) + pack_file = Path.join(pack_path, "pack.json") + + {pack_name, Jason.decode!(File.read!(pack_file))} + end + + defp validate_pack({name, pack}) do + pack_path = Path.join(@emoji_dir_path, name) + + if can_download?(pack, pack_path) do + archive_for_sha = make_archive(name, pack, pack_path) + archive_sha = :crypto.hash(:sha256, archive_for_sha) |> Base.encode16() + + pack = + pack + |> put_in(["pack", "can-download"], true) + |> put_in(["pack", "download-sha256"], archive_sha) + + {name, pack} + else + {name, put_in(pack, ["pack", "can-download"], false)} + end + end + + defp can_download?(pack, pack_path) do + # If the pack is set as shared, check if it can be downloaded + # That means that when asked, the pack can be packed and sent to the remote + # Otherwise, they'd have to download it from external-src + pack["pack"]["share-files"] && + Enum.all?(pack["files"], fn {_, path} -> + File.exists?(Path.join(pack_path, path)) + end) + end + + defp create_archive_and_cache(name, pack, pack_dir, md5) do + files = + ['pack.json'] ++ + (pack["files"] |> Enum.map(fn {_, path} -> to_charlist(path) end)) + + {:ok, {_, zip_result}} = :zip.zip('#{name}.zip', files, [:memory, cwd: to_charlist(pack_dir)]) + + cache_ms = :timer.seconds(@cache_seconds_per_file * Enum.count(files)) + + Cachex.put!( + :emoji_packs_cache, + name, + # if pack.json MD5 changes, the cache is not valid anymore + %{pack_json_md5: md5, pack_data: zip_result}, + # Add a minute to cache time for every file in the pack + ttl: cache_ms + ) + + Logger.debug("Created an archive for the '#{name}' emoji pack, \ +keeping it in cache for #{div(cache_ms, 1000)}s") + + zip_result + end + + defp make_archive(name, pack, pack_dir) do + # Having a different pack.json md5 invalidates cache + pack_file_md5 = :crypto.hash(:md5, File.read!(Path.join(pack_dir, "pack.json"))) + + case Cachex.get!(:emoji_packs_cache, name) do + %{pack_file_md5: ^pack_file_md5, pack_data: zip_result} -> + Logger.debug("Using cache for the '#{name}' shared emoji pack") + zip_result + + _ -> + create_archive_and_cache(name, pack, pack_dir, pack_file_md5) + end + end + + @doc """ + An endpoint for other instances (via admin UI) or users (via browser) + to download packs that the instance shares. + """ + def download_shared(conn, %{"name" => name}) do + pack_dir = Path.join(@emoji_dir_path, name) + pack_file = Path.join(pack_dir, "pack.json") + + with {_, true} <- {:exists?, File.exists?(pack_file)}, + pack = Jason.decode!(File.read!(pack_file)), + {_, true} <- {:can_download?, can_download?(pack, pack_dir)} do + zip_result = make_archive(name, pack, pack_dir) + send_download(conn, {:binary, zip_result}, filename: "#{name}.zip") + else + {:can_download?, _} -> + conn + |> put_status(:forbidden) + |> json(%{ + error: "Pack #{name} cannot be downloaded from this instance, either pack sharing\ + was disabled for this pack or some files are missing" + }) + + {:exists?, _} -> + conn + |> put_status(:not_found) + |> json(%{error: "Pack #{name} does not exist"}) + end + end + + @doc """ + An admin endpoint to request downloading a pack named `pack_name` from the instance + `instance_address`. + + If the requested instance's admin chose to share the pack, it will be downloaded + from that instance, otherwise it will be downloaded from the fallback source, if there is one. + """ + def download_from(conn, %{"instance_address" => address, "pack_name" => name} = data) do + shareable_packs_available = + "#{address}/nodeinfo/2.1.json" + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> Map.get("features") + |> Enum.member?("shareable_emoji_packs") + + if shareable_packs_available do + full_pack = + "#{address}/api/pleroma/emoji/packs/list" + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> Map.get(name) + + pack_info_res = + case full_pack["pack"] do + %{"share-files" => true, "can-download" => true, "download-sha256" => sha} -> + {:ok, + %{ + sha: sha, + uri: "#{address}/api/pleroma/emoji/packs/download_shared/#{name}" + }} + + %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) -> + {:ok, + %{ + sha: sha, + uri: src, + fallback: true + }} + + _ -> + {:error, + "The pack was not set as shared and there is no fallback src to download from"} + end + + with {:ok, %{sha: sha, uri: uri} = pinfo} <- pack_info_res, + %{body: emoji_archive} <- Tesla.get!(uri), + {_, true} <- {:checksum, Base.decode16!(sha) == :crypto.hash(:sha256, emoji_archive)} do + local_name = data["as"] || name + pack_dir = Path.join(@emoji_dir_path, local_name) + File.mkdir_p!(pack_dir) + + files = Enum.map(full_pack["files"], fn {_, path} -> to_charlist(path) end) + # Fallback cannot contain a pack.json file + files = if pinfo[:fallback], do: files, else: ['pack.json'] ++ files + + {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) + + # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256 + # in it to depend on itself + if pinfo[:fallback] do + pack_file_path = Path.join(pack_dir, "pack.json") + + File.write!(pack_file_path, Jason.encode!(full_pack, pretty: true)) + end + + json(conn, "ok") + else + {:error, e} -> + conn |> put_status(:internal_server_error) |> json(%{error: e}) + + {:checksum, _} -> + conn + |> put_status(:internal_server_error) + |> json(%{error: "SHA256 for the pack doesn't match the one sent by the server"}) + end + else + conn + |> put_status(:internal_server_error) + |> json(%{error: "The requested instance does not support sharing emoji packs"}) + end + end + + @doc """ + Creates an empty pack named `name` which then can be updated via the admin UI. + """ + def create(conn, %{"name" => name}) do + pack_dir = Path.join(@emoji_dir_path, name) + + if not File.exists?(pack_dir) do + File.mkdir_p!(pack_dir) + + pack_file_p = Path.join(pack_dir, "pack.json") + + File.write!( + pack_file_p, + Jason.encode!(%{pack: %{}, files: %{}}) + ) + + conn |> json("ok") + else + conn + |> put_status(:conflict) + |> json(%{error: "A pack named \"#{name}\" already exists"}) + end + end + + @doc """ + Deletes the pack `name` and all it's files. + """ + def delete(conn, %{"name" => name}) do + pack_dir = Path.join(@emoji_dir_path, name) + + case File.rm_rf(pack_dir) do + {:ok, _} -> + conn |> json("ok") + + {:error, _} -> + conn + |> put_status(:internal_server_error) + |> json(%{error: "Couldn't delete the pack #{name}"}) + end + end + + @doc """ + An endpoint to update `pack_names`'s metadata. + + `new_data` is the new metadata for the pack, that will replace the old metadata. + """ + def update_metadata(conn, %{"pack_name" => name, "new_data" => new_data}) do + pack_file_p = Path.join([@emoji_dir_path, name, "pack.json"]) + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + # The new fallback-src is in the new data and it's not the same as it was in the old data + should_update_fb_sha = + not is_nil(new_data["fallback-src"]) and + new_data["fallback-src"] != full_pack["pack"]["fallback-src"] + + with {_, true} <- {:should_update?, should_update_fb_sha}, + %{body: pack_arch} <- Tesla.get!(new_data["fallback-src"]), + {:ok, flist} <- :zip.unzip(pack_arch, [:memory]), + {_, true} <- {:has_all_files?, has_all_files?(full_pack, flist)} do + fallback_sha = :crypto.hash(:sha256, pack_arch) |> Base.encode16() + + new_data = Map.put(new_data, "fallback-src-sha256", fallback_sha) + update_metadata_and_send(conn, full_pack, new_data, pack_file_p) + else + {:should_update?, _} -> + update_metadata_and_send(conn, full_pack, new_data, pack_file_p) + + {:has_all_files?, _} -> + conn + |> put_status(:bad_request) + |> json(%{error: "The fallback archive does not have all files specified in pack.json"}) + end + end + + # Check if all files from the pack.json are in the archive + defp has_all_files?(%{"files" => files}, flist) do + Enum.all?(files, fn {_, from_manifest} -> + Enum.find(flist, fn {from_archive, _} -> + to_string(from_archive) == from_manifest + end) + end) + end + + defp update_metadata_and_send(conn, full_pack, new_data, pack_file_p) do + full_pack = Map.put(full_pack, "pack", new_data) + File.write!(pack_file_p, Jason.encode!(full_pack, pretty: true)) + + # Send new data back with fallback sha filled + json(conn, new_data) + end + + defp get_filename(%{"filename" => filename}), do: filename + + defp get_filename(%{"file" => file}) do + case file do + %Plug.Upload{filename: filename} -> filename + url when is_binary(url) -> Path.basename(url) + end + end + + defp empty?(str), do: String.trim(str) == "" + + defp update_file_and_send(conn, updated_full_pack, pack_file_p) do + # Write the emoji pack file + File.write!(pack_file_p, Jason.encode!(updated_full_pack, pretty: true)) + + # Return the modified file list + json(conn, updated_full_pack["files"]) + end + + @doc """ + Updates a file in a pack. + + Updating can mean three things: + + - `add` adds an emoji named `shortcode` to the pack `pack_name`, + that means that the emoji file needs to be uploaded with the request + (thus requiring it to be a multipart request) and be named `file`. + There can also be an optional `filename` that will be the new emoji file name + (if it's not there, the name will be taken from the uploaded file). + - `update` changes emoji shortcode (from `shortcode` to `new_shortcode` or moves the file + (from the current filename to `new_filename`) + - `remove` removes the emoji named `shortcode` and it's associated file + """ + + # Add + def update_file( + conn, + %{"pack_name" => pack_name, "action" => "add", "shortcode" => shortcode} = params + ) do + pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_file_p = Path.join(pack_dir, "pack.json") + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + with {_, false} <- {:has_shortcode, Map.has_key?(full_pack["files"], shortcode)}, + filename <- get_filename(params), + false <- empty?(shortcode), + false <- empty?(filename) do + file_path = Path.join(pack_dir, filename) + + # If the name contains directories, create them + if String.contains?(file_path, "/") do + File.mkdir_p!(Path.dirname(file_path)) + end + + case params["file"] do + %Plug.Upload{path: upload_path} -> + # Copy the uploaded file from the temporary directory + File.copy!(upload_path, file_path) + + url when is_binary(url) -> + # Download and write the file + file_contents = Tesla.get!(url).body + File.write!(file_path, file_contents) + end + + updated_full_pack = put_in(full_pack, ["files", shortcode], filename) + update_file_and_send(conn, updated_full_pack, pack_file_p) + else + {:has_shortcode, _} -> + conn + |> put_status(:conflict) + |> json(%{error: "An emoji with the \"#{shortcode}\" shortcode already exists"}) + + true -> + conn + |> put_status(:bad_request) + |> json(%{error: "shortcode or filename cannot be empty"}) + end + end + + # Remove + def update_file(conn, %{ + "pack_name" => pack_name, + "action" => "remove", + "shortcode" => shortcode + }) do + pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_file_p = Path.join(pack_dir, "pack.json") + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + if Map.has_key?(full_pack["files"], shortcode) do + {emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) + + emoji_file_path = Path.join(pack_dir, emoji_file_path) + + # Delete the emoji file + File.rm!(emoji_file_path) + + # If the old directory has no more files, remove it + if String.contains?(emoji_file_path, "/") do + dir = Path.dirname(emoji_file_path) + + if Enum.empty?(File.ls!(dir)) do + File.rmdir!(dir) + end + end + + update_file_and_send(conn, updated_full_pack, pack_file_p) + else + conn + |> put_status(:bad_request) + |> json(%{error: "Emoji \"#{shortcode}\" does not exist"}) + end + end + + # Update + def update_file( + conn, + %{"pack_name" => pack_name, "action" => "update", "shortcode" => shortcode} = params + ) do + pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_file_p = Path.join(pack_dir, "pack.json") + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + with {_, true} <- {:has_shortcode, Map.has_key?(full_pack["files"], shortcode)}, + %{"new_shortcode" => new_shortcode, "new_filename" => new_filename} <- params, + false <- empty?(new_shortcode), + false <- empty?(new_filename) do + # First, remove the old shortcode, saving the old path + {old_emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) + old_emoji_file_path = Path.join(pack_dir, old_emoji_file_path) + new_emoji_file_path = Path.join(pack_dir, new_filename) + + # If the name contains directories, create them + if String.contains?(new_emoji_file_path, "/") do + File.mkdir_p!(Path.dirname(new_emoji_file_path)) + end + + # Move/Rename the old filename to a new filename + # These are probably on the same filesystem, so just rename should work + :ok = File.rename(old_emoji_file_path, new_emoji_file_path) + + # If the old directory has no more files, remove it + if String.contains?(old_emoji_file_path, "/") do + dir = Path.dirname(old_emoji_file_path) + + if Enum.empty?(File.ls!(dir)) do + File.rmdir!(dir) + end + end + + # Then, put in the new shortcode with the new path + updated_full_pack = put_in(updated_full_pack, ["files", new_shortcode], new_filename) + update_file_and_send(conn, updated_full_pack, pack_file_p) + else + {:has_shortcode, _} -> + conn + |> put_status(:bad_request) + |> json(%{error: "Emoji \"#{shortcode}\" does not exist"}) + + true -> + conn + |> put_status(:bad_request) + |> json(%{error: "new_shortcode or new_filename cannot be empty"}) + + _ -> + conn + |> put_status(:bad_request) + |> json(%{error: "new_shortcode or new_file were not specified"}) + end + end + + def update_file(conn, %{"action" => action}) do + conn + |> put_status(:bad_request) + |> json(%{error: "Unknown action: #{action}"}) + end + + @doc """ + Imports emoji from the filesystem. + + Importing means checking all the directories in the + `$instance_static/emoji/` for directories which do not have + `pack.json`. If one has an emoji.txt file, that file will be used + to create a `pack.json` file with it's contents. If the directory has + neither, all the files with specific configured extenstions will be + assumed to be emojis and stored in the new `pack.json` file. + """ + def import_from_fs(conn, _params) do + with {:ok, results} <- File.ls(@emoji_dir_path) do + imported_pack_names = + results + |> Enum.filter(fn file -> + dir_path = Path.join(@emoji_dir_path, file) + # Find the directories that do NOT have pack.json + File.dir?(dir_path) and not File.exists?(Path.join(dir_path, "pack.json")) + end) + |> Enum.map(&write_pack_json_contents/1) + + json(conn, imported_pack_names) + else + {:error, _} -> + conn + |> put_status(:internal_server_error) + |> json(%{error: "Error accessing emoji pack directory"}) + end + end + + defp write_pack_json_contents(dir) do + dir_path = Path.join(@emoji_dir_path, dir) + emoji_txt_path = Path.join(dir_path, "emoji.txt") + + files_for_pack = files_for_pack(emoji_txt_path, dir_path) + pack_json_contents = Jason.encode!(%{pack: %{}, files: files_for_pack}) + + File.write!(Path.join(dir_path, "pack.json"), pack_json_contents) + + dir + end + + defp files_for_pack(emoji_txt_path, dir_path) do + if File.exists?(emoji_txt_path) do + # There's an emoji.txt file, it's likely from a pack installed by the pack manager. + # Make a pack.json file from the contents of that emoji.txt fileh + + # FIXME: Copy-pasted from Pleroma.Emoji/load_from_file_stream/2 + + # Create a map of shortcodes to filenames from emoji.txt + File.read!(emoji_txt_path) + |> String.split("\n") + |> Enum.map(&String.trim/1) + |> Enum.map(fn line -> + case String.split(line, ~r/,\s*/) do + # This matches both strings with and without tags + # and we don't care about tags here + [name, file | _] -> {name, file} + _ -> nil + end + end) + |> Enum.filter(fn x -> not is_nil(x) end) + |> Enum.into(%{}) + else + # If there's no emoji.txt, assume all files + # that are of certain extensions from the config are emojis and import them all + pack_extensions = Pleroma.Config.get!([:emoji, :pack_extensions]) + Pleroma.Emoji.make_shortcode_to_file_map(dir_path, pack_extensions) + end + end +end -- cgit v1.2.3 From d51e5e447ee944e77646b15a7aabc0214e99c351 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 12 Sep 2019 20:38:57 +0300 Subject: Move emoji reloading to admin api --- lib/pleroma/web/admin_api/admin_api_controller.ex | 6 ++++++ lib/pleroma/web/pleroma_api/emoji_api_controller.ex | 6 ------ lib/pleroma/web/router.ex | 8 ++------ 3 files changed, 8 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 8a8091daa..4d4e862dd 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -599,6 +599,12 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do |> render("index.json", %{configs: updated}) end + def reload_emoji(conn, _params) do + Pleroma.Emoji.reload() + + conn |> json("ok") + end + def errors(conn, {:error, :not_found}) do conn |> put_status(:not_found) diff --git a/lib/pleroma/web/pleroma_api/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/emoji_api_controller.ex index 36ca2c804..bc1639095 100644 --- a/lib/pleroma/web/pleroma_api/emoji_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/emoji_api_controller.ex @@ -3,12 +3,6 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do require Logger - def reload(conn, _params) do - Pleroma.Emoji.reload() - - conn |> json("ok") - end - @emoji_dir_path Path.join( Pleroma.Config.get!([:instance, :static_dir]), "emoji" diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 715e4ba68..71ef382c5 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -205,15 +205,11 @@ defmodule Pleroma.Web.Router do get("/config/migrate_from_db", AdminAPIController, :migrate_from_db) get("/moderation_log", AdminAPIController, :list_log) + + post("/reload_emoji", AdminAPIController, :reload_emoji) end scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do - scope [] do - pipe_through([:admin_api, :oauth_write]) - - post("/reload", EmojiAPIController, :reload) - end - scope "/packs" do # Modifying packs pipe_through([:admin_api, :oauth_write]) -- cgit v1.2.3 From a1325d5fd9b540017cbffbb73db85ee9fa9f12d0 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 18 Sep 2019 18:09:57 +0300 Subject: Change path from nodeinfo to metadata->features --- lib/pleroma/web/pleroma_api/emoji_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/pleroma_api/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/emoji_api_controller.ex index bc1639095..391c317e7 100644 --- a/lib/pleroma/web/pleroma_api/emoji_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/emoji_api_controller.ex @@ -152,7 +152,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") |> Tesla.get!() |> Map.get(:body) |> Jason.decode!() - |> Map.get("features") + |> get_in(["metadata", "features"]) |> Enum.member?("shareable_emoji_packs") if shareable_packs_available do -- cgit v1.2.3 From b585134c9092b49e7b5c24e04d6d6315d45dd0a2 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 18 Sep 2019 19:48:25 +0300 Subject: Get the nodeinfo address from the well-known --- lib/pleroma/web/pleroma_api/emoji_api_controller.ex | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/pleroma_api/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/emoji_api_controller.ex index 391c317e7..6beca426a 100644 --- a/lib/pleroma/web/pleroma_api/emoji_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/emoji_api_controller.ex @@ -148,7 +148,13 @@ keeping it in cache for #{div(cache_ms, 1000)}s") """ def download_from(conn, %{"instance_address" => address, "pack_name" => name} = data) do shareable_packs_available = - "#{address}/nodeinfo/2.1.json" + "#{address}/.well-known/nodeinfo" + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> List.last() + |> Map.get("href") + # Get the actual nodeinfo address and fetch it |> Tesla.get!() |> Map.get(:body) |> Jason.decode!() -- cgit v1.2.3 From fe5e0b784604b1352e98e7915c3c67d59ac4f709 Mon Sep 17 00:00:00 2001 From: eugenijm Date: Thu, 19 Sep 2019 08:27:55 +0300 Subject: Mastodon API: Return `pleroma.direct_conversation_id` when creating direct messages (`POST /api/v1/statuses`) --- .../web/mastodon_api/controllers/mastodon_api_controller.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index 37eeb2ac3..6704ee7e8 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -611,7 +611,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do {:ok, activity} -> conn |> put_view(StatusView) - |> try_render("status.json", %{activity: activity, for: user, as: :activity}) + |> try_render("status.json", %{ + activity: activity, + for: user, + as: :activity, + with_direct_conversation_id: true + }) end end end -- cgit v1.2.3 From 0e6085da106cb966c340fac2d307d9e8e26e91ed Mon Sep 17 00:00:00 2001 From: D Anzorge Date: Thu, 19 Sep 2019 16:09:07 +0200 Subject: Fix pagination in AP outbox.json --- lib/pleroma/web/activity_pub/views/user_view.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 164b973d0..a2f73e140 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -227,11 +227,12 @@ defmodule Pleroma.Web.ActivityPub.UserView do activities = ActivityPub.fetch_user_activities(user, nil, params) + # this is sorted chronologically, so first activity is the newest (max) {max_id, min_id, collection} = if length(activities) > 0 do { - Enum.at(Enum.reverse(activities), 0).id, Enum.at(activities, 0).id, + Enum.at(Enum.reverse(activities), 0).id, Enum.map(activities, fn act -> {:ok, data} = Transmogrifier.prepare_outgoing(act.data) data -- cgit v1.2.3 From 6b3d5ed6db6a3c73eb1f8373ebd670427aa8849d Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 23 Sep 2019 21:14:51 +0300 Subject: Emoji API Controller: Follow phoenix directory structure --- .../controllers/emoji_api_controller.ex | 575 +++++++++++++++++++++ .../controllers/pleroma_api_controller.ex | 89 ++++ .../web/pleroma_api/emoji_api_controller.ex | 575 --------------------- .../web/pleroma_api/pleroma_api_controller.ex | 89 ---- 4 files changed, 664 insertions(+), 664 deletions(-) create mode 100644 lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex create mode 100644 lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex delete mode 100644 lib/pleroma/web/pleroma_api/emoji_api_controller.ex delete mode 100644 lib/pleroma/web/pleroma_api/pleroma_api_controller.ex (limited to 'lib') diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex new file mode 100644 index 000000000..6beca426a --- /dev/null +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex @@ -0,0 +1,575 @@ +defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do + use Pleroma.Web, :controller + + require Logger + + @emoji_dir_path Path.join( + Pleroma.Config.get!([:instance, :static_dir]), + "emoji" + ) + + @cache_seconds_per_file Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file]) + + @doc """ + Lists the packs available on the instance as JSON. + + The information is public and does not require authentification. The format is + a map of "pack directory name" to pack.json contents. + """ + def list_packs(conn, _params) do + with {:ok, results} <- File.ls(@emoji_dir_path) do + pack_infos = + results + |> Enum.filter(&has_pack_json?/1) + |> Enum.map(&load_pack/1) + # Check if all the files are in place and can be sent + |> Enum.map(&validate_pack/1) + # Transform into a map of pack-name => pack-data + |> Enum.into(%{}) + + json(conn, pack_infos) + end + end + + defp has_pack_json?(file) do + dir_path = Path.join(@emoji_dir_path, file) + # Filter to only use the pack.json packs + File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.json")) + end + + defp load_pack(pack_name) do + pack_path = Path.join(@emoji_dir_path, pack_name) + pack_file = Path.join(pack_path, "pack.json") + + {pack_name, Jason.decode!(File.read!(pack_file))} + end + + defp validate_pack({name, pack}) do + pack_path = Path.join(@emoji_dir_path, name) + + if can_download?(pack, pack_path) do + archive_for_sha = make_archive(name, pack, pack_path) + archive_sha = :crypto.hash(:sha256, archive_for_sha) |> Base.encode16() + + pack = + pack + |> put_in(["pack", "can-download"], true) + |> put_in(["pack", "download-sha256"], archive_sha) + + {name, pack} + else + {name, put_in(pack, ["pack", "can-download"], false)} + end + end + + defp can_download?(pack, pack_path) do + # If the pack is set as shared, check if it can be downloaded + # That means that when asked, the pack can be packed and sent to the remote + # Otherwise, they'd have to download it from external-src + pack["pack"]["share-files"] && + Enum.all?(pack["files"], fn {_, path} -> + File.exists?(Path.join(pack_path, path)) + end) + end + + defp create_archive_and_cache(name, pack, pack_dir, md5) do + files = + ['pack.json'] ++ + (pack["files"] |> Enum.map(fn {_, path} -> to_charlist(path) end)) + + {:ok, {_, zip_result}} = :zip.zip('#{name}.zip', files, [:memory, cwd: to_charlist(pack_dir)]) + + cache_ms = :timer.seconds(@cache_seconds_per_file * Enum.count(files)) + + Cachex.put!( + :emoji_packs_cache, + name, + # if pack.json MD5 changes, the cache is not valid anymore + %{pack_json_md5: md5, pack_data: zip_result}, + # Add a minute to cache time for every file in the pack + ttl: cache_ms + ) + + Logger.debug("Created an archive for the '#{name}' emoji pack, \ +keeping it in cache for #{div(cache_ms, 1000)}s") + + zip_result + end + + defp make_archive(name, pack, pack_dir) do + # Having a different pack.json md5 invalidates cache + pack_file_md5 = :crypto.hash(:md5, File.read!(Path.join(pack_dir, "pack.json"))) + + case Cachex.get!(:emoji_packs_cache, name) do + %{pack_file_md5: ^pack_file_md5, pack_data: zip_result} -> + Logger.debug("Using cache for the '#{name}' shared emoji pack") + zip_result + + _ -> + create_archive_and_cache(name, pack, pack_dir, pack_file_md5) + end + end + + @doc """ + An endpoint for other instances (via admin UI) or users (via browser) + to download packs that the instance shares. + """ + def download_shared(conn, %{"name" => name}) do + pack_dir = Path.join(@emoji_dir_path, name) + pack_file = Path.join(pack_dir, "pack.json") + + with {_, true} <- {:exists?, File.exists?(pack_file)}, + pack = Jason.decode!(File.read!(pack_file)), + {_, true} <- {:can_download?, can_download?(pack, pack_dir)} do + zip_result = make_archive(name, pack, pack_dir) + send_download(conn, {:binary, zip_result}, filename: "#{name}.zip") + else + {:can_download?, _} -> + conn + |> put_status(:forbidden) + |> json(%{ + error: "Pack #{name} cannot be downloaded from this instance, either pack sharing\ + was disabled for this pack or some files are missing" + }) + + {:exists?, _} -> + conn + |> put_status(:not_found) + |> json(%{error: "Pack #{name} does not exist"}) + end + end + + @doc """ + An admin endpoint to request downloading a pack named `pack_name` from the instance + `instance_address`. + + If the requested instance's admin chose to share the pack, it will be downloaded + from that instance, otherwise it will be downloaded from the fallback source, if there is one. + """ + def download_from(conn, %{"instance_address" => address, "pack_name" => name} = data) do + shareable_packs_available = + "#{address}/.well-known/nodeinfo" + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> List.last() + |> Map.get("href") + # Get the actual nodeinfo address and fetch it + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> get_in(["metadata", "features"]) + |> Enum.member?("shareable_emoji_packs") + + if shareable_packs_available do + full_pack = + "#{address}/api/pleroma/emoji/packs/list" + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> Map.get(name) + + pack_info_res = + case full_pack["pack"] do + %{"share-files" => true, "can-download" => true, "download-sha256" => sha} -> + {:ok, + %{ + sha: sha, + uri: "#{address}/api/pleroma/emoji/packs/download_shared/#{name}" + }} + + %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) -> + {:ok, + %{ + sha: sha, + uri: src, + fallback: true + }} + + _ -> + {:error, + "The pack was not set as shared and there is no fallback src to download from"} + end + + with {:ok, %{sha: sha, uri: uri} = pinfo} <- pack_info_res, + %{body: emoji_archive} <- Tesla.get!(uri), + {_, true} <- {:checksum, Base.decode16!(sha) == :crypto.hash(:sha256, emoji_archive)} do + local_name = data["as"] || name + pack_dir = Path.join(@emoji_dir_path, local_name) + File.mkdir_p!(pack_dir) + + files = Enum.map(full_pack["files"], fn {_, path} -> to_charlist(path) end) + # Fallback cannot contain a pack.json file + files = if pinfo[:fallback], do: files, else: ['pack.json'] ++ files + + {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) + + # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256 + # in it to depend on itself + if pinfo[:fallback] do + pack_file_path = Path.join(pack_dir, "pack.json") + + File.write!(pack_file_path, Jason.encode!(full_pack, pretty: true)) + end + + json(conn, "ok") + else + {:error, e} -> + conn |> put_status(:internal_server_error) |> json(%{error: e}) + + {:checksum, _} -> + conn + |> put_status(:internal_server_error) + |> json(%{error: "SHA256 for the pack doesn't match the one sent by the server"}) + end + else + conn + |> put_status(:internal_server_error) + |> json(%{error: "The requested instance does not support sharing emoji packs"}) + end + end + + @doc """ + Creates an empty pack named `name` which then can be updated via the admin UI. + """ + def create(conn, %{"name" => name}) do + pack_dir = Path.join(@emoji_dir_path, name) + + if not File.exists?(pack_dir) do + File.mkdir_p!(pack_dir) + + pack_file_p = Path.join(pack_dir, "pack.json") + + File.write!( + pack_file_p, + Jason.encode!(%{pack: %{}, files: %{}}) + ) + + conn |> json("ok") + else + conn + |> put_status(:conflict) + |> json(%{error: "A pack named \"#{name}\" already exists"}) + end + end + + @doc """ + Deletes the pack `name` and all it's files. + """ + def delete(conn, %{"name" => name}) do + pack_dir = Path.join(@emoji_dir_path, name) + + case File.rm_rf(pack_dir) do + {:ok, _} -> + conn |> json("ok") + + {:error, _} -> + conn + |> put_status(:internal_server_error) + |> json(%{error: "Couldn't delete the pack #{name}"}) + end + end + + @doc """ + An endpoint to update `pack_names`'s metadata. + + `new_data` is the new metadata for the pack, that will replace the old metadata. + """ + def update_metadata(conn, %{"pack_name" => name, "new_data" => new_data}) do + pack_file_p = Path.join([@emoji_dir_path, name, "pack.json"]) + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + # The new fallback-src is in the new data and it's not the same as it was in the old data + should_update_fb_sha = + not is_nil(new_data["fallback-src"]) and + new_data["fallback-src"] != full_pack["pack"]["fallback-src"] + + with {_, true} <- {:should_update?, should_update_fb_sha}, + %{body: pack_arch} <- Tesla.get!(new_data["fallback-src"]), + {:ok, flist} <- :zip.unzip(pack_arch, [:memory]), + {_, true} <- {:has_all_files?, has_all_files?(full_pack, flist)} do + fallback_sha = :crypto.hash(:sha256, pack_arch) |> Base.encode16() + + new_data = Map.put(new_data, "fallback-src-sha256", fallback_sha) + update_metadata_and_send(conn, full_pack, new_data, pack_file_p) + else + {:should_update?, _} -> + update_metadata_and_send(conn, full_pack, new_data, pack_file_p) + + {:has_all_files?, _} -> + conn + |> put_status(:bad_request) + |> json(%{error: "The fallback archive does not have all files specified in pack.json"}) + end + end + + # Check if all files from the pack.json are in the archive + defp has_all_files?(%{"files" => files}, flist) do + Enum.all?(files, fn {_, from_manifest} -> + Enum.find(flist, fn {from_archive, _} -> + to_string(from_archive) == from_manifest + end) + end) + end + + defp update_metadata_and_send(conn, full_pack, new_data, pack_file_p) do + full_pack = Map.put(full_pack, "pack", new_data) + File.write!(pack_file_p, Jason.encode!(full_pack, pretty: true)) + + # Send new data back with fallback sha filled + json(conn, new_data) + end + + defp get_filename(%{"filename" => filename}), do: filename + + defp get_filename(%{"file" => file}) do + case file do + %Plug.Upload{filename: filename} -> filename + url when is_binary(url) -> Path.basename(url) + end + end + + defp empty?(str), do: String.trim(str) == "" + + defp update_file_and_send(conn, updated_full_pack, pack_file_p) do + # Write the emoji pack file + File.write!(pack_file_p, Jason.encode!(updated_full_pack, pretty: true)) + + # Return the modified file list + json(conn, updated_full_pack["files"]) + end + + @doc """ + Updates a file in a pack. + + Updating can mean three things: + + - `add` adds an emoji named `shortcode` to the pack `pack_name`, + that means that the emoji file needs to be uploaded with the request + (thus requiring it to be a multipart request) and be named `file`. + There can also be an optional `filename` that will be the new emoji file name + (if it's not there, the name will be taken from the uploaded file). + - `update` changes emoji shortcode (from `shortcode` to `new_shortcode` or moves the file + (from the current filename to `new_filename`) + - `remove` removes the emoji named `shortcode` and it's associated file + """ + + # Add + def update_file( + conn, + %{"pack_name" => pack_name, "action" => "add", "shortcode" => shortcode} = params + ) do + pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_file_p = Path.join(pack_dir, "pack.json") + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + with {_, false} <- {:has_shortcode, Map.has_key?(full_pack["files"], shortcode)}, + filename <- get_filename(params), + false <- empty?(shortcode), + false <- empty?(filename) do + file_path = Path.join(pack_dir, filename) + + # If the name contains directories, create them + if String.contains?(file_path, "/") do + File.mkdir_p!(Path.dirname(file_path)) + end + + case params["file"] do + %Plug.Upload{path: upload_path} -> + # Copy the uploaded file from the temporary directory + File.copy!(upload_path, file_path) + + url when is_binary(url) -> + # Download and write the file + file_contents = Tesla.get!(url).body + File.write!(file_path, file_contents) + end + + updated_full_pack = put_in(full_pack, ["files", shortcode], filename) + update_file_and_send(conn, updated_full_pack, pack_file_p) + else + {:has_shortcode, _} -> + conn + |> put_status(:conflict) + |> json(%{error: "An emoji with the \"#{shortcode}\" shortcode already exists"}) + + true -> + conn + |> put_status(:bad_request) + |> json(%{error: "shortcode or filename cannot be empty"}) + end + end + + # Remove + def update_file(conn, %{ + "pack_name" => pack_name, + "action" => "remove", + "shortcode" => shortcode + }) do + pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_file_p = Path.join(pack_dir, "pack.json") + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + if Map.has_key?(full_pack["files"], shortcode) do + {emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) + + emoji_file_path = Path.join(pack_dir, emoji_file_path) + + # Delete the emoji file + File.rm!(emoji_file_path) + + # If the old directory has no more files, remove it + if String.contains?(emoji_file_path, "/") do + dir = Path.dirname(emoji_file_path) + + if Enum.empty?(File.ls!(dir)) do + File.rmdir!(dir) + end + end + + update_file_and_send(conn, updated_full_pack, pack_file_p) + else + conn + |> put_status(:bad_request) + |> json(%{error: "Emoji \"#{shortcode}\" does not exist"}) + end + end + + # Update + def update_file( + conn, + %{"pack_name" => pack_name, "action" => "update", "shortcode" => shortcode} = params + ) do + pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_file_p = Path.join(pack_dir, "pack.json") + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + with {_, true} <- {:has_shortcode, Map.has_key?(full_pack["files"], shortcode)}, + %{"new_shortcode" => new_shortcode, "new_filename" => new_filename} <- params, + false <- empty?(new_shortcode), + false <- empty?(new_filename) do + # First, remove the old shortcode, saving the old path + {old_emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) + old_emoji_file_path = Path.join(pack_dir, old_emoji_file_path) + new_emoji_file_path = Path.join(pack_dir, new_filename) + + # If the name contains directories, create them + if String.contains?(new_emoji_file_path, "/") do + File.mkdir_p!(Path.dirname(new_emoji_file_path)) + end + + # Move/Rename the old filename to a new filename + # These are probably on the same filesystem, so just rename should work + :ok = File.rename(old_emoji_file_path, new_emoji_file_path) + + # If the old directory has no more files, remove it + if String.contains?(old_emoji_file_path, "/") do + dir = Path.dirname(old_emoji_file_path) + + if Enum.empty?(File.ls!(dir)) do + File.rmdir!(dir) + end + end + + # Then, put in the new shortcode with the new path + updated_full_pack = put_in(updated_full_pack, ["files", new_shortcode], new_filename) + update_file_and_send(conn, updated_full_pack, pack_file_p) + else + {:has_shortcode, _} -> + conn + |> put_status(:bad_request) + |> json(%{error: "Emoji \"#{shortcode}\" does not exist"}) + + true -> + conn + |> put_status(:bad_request) + |> json(%{error: "new_shortcode or new_filename cannot be empty"}) + + _ -> + conn + |> put_status(:bad_request) + |> json(%{error: "new_shortcode or new_file were not specified"}) + end + end + + def update_file(conn, %{"action" => action}) do + conn + |> put_status(:bad_request) + |> json(%{error: "Unknown action: #{action}"}) + end + + @doc """ + Imports emoji from the filesystem. + + Importing means checking all the directories in the + `$instance_static/emoji/` for directories which do not have + `pack.json`. If one has an emoji.txt file, that file will be used + to create a `pack.json` file with it's contents. If the directory has + neither, all the files with specific configured extenstions will be + assumed to be emojis and stored in the new `pack.json` file. + """ + def import_from_fs(conn, _params) do + with {:ok, results} <- File.ls(@emoji_dir_path) do + imported_pack_names = + results + |> Enum.filter(fn file -> + dir_path = Path.join(@emoji_dir_path, file) + # Find the directories that do NOT have pack.json + File.dir?(dir_path) and not File.exists?(Path.join(dir_path, "pack.json")) + end) + |> Enum.map(&write_pack_json_contents/1) + + json(conn, imported_pack_names) + else + {:error, _} -> + conn + |> put_status(:internal_server_error) + |> json(%{error: "Error accessing emoji pack directory"}) + end + end + + defp write_pack_json_contents(dir) do + dir_path = Path.join(@emoji_dir_path, dir) + emoji_txt_path = Path.join(dir_path, "emoji.txt") + + files_for_pack = files_for_pack(emoji_txt_path, dir_path) + pack_json_contents = Jason.encode!(%{pack: %{}, files: files_for_pack}) + + File.write!(Path.join(dir_path, "pack.json"), pack_json_contents) + + dir + end + + defp files_for_pack(emoji_txt_path, dir_path) do + if File.exists?(emoji_txt_path) do + # There's an emoji.txt file, it's likely from a pack installed by the pack manager. + # Make a pack.json file from the contents of that emoji.txt fileh + + # FIXME: Copy-pasted from Pleroma.Emoji/load_from_file_stream/2 + + # Create a map of shortcodes to filenames from emoji.txt + File.read!(emoji_txt_path) + |> String.split("\n") + |> Enum.map(&String.trim/1) + |> Enum.map(fn line -> + case String.split(line, ~r/,\s*/) do + # This matches both strings with and without tags + # and we don't care about tags here + [name, file | _] -> {name, file} + _ -> nil + end + end) + |> Enum.filter(fn x -> not is_nil(x) end) + |> Enum.into(%{}) + else + # If there's no emoji.txt, assume all files + # that are of certain extensions from the config are emojis and import them all + pack_extensions = Pleroma.Config.get!([:emoji, :pack_extensions]) + Pleroma.Emoji.make_shortcode_to_file_map(dir_path, pack_extensions) + end + end +end diff --git a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex new file mode 100644 index 000000000..d17ccf84d --- /dev/null +++ b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex @@ -0,0 +1,89 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do + use Pleroma.Web, :controller + + import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2] + + alias Pleroma.Conversation.Participation + alias Pleroma.Notification + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.MastodonAPI.ConversationView + alias Pleroma.Web.MastodonAPI.NotificationView + alias Pleroma.Web.MastodonAPI.StatusView + + def conversation(%{assigns: %{user: user}} = conn, %{"id" => participation_id}) do + with %Participation{} = participation <- Participation.get(participation_id), + true <- user.id == participation.user_id do + conn + |> put_view(ConversationView) + |> render("participation.json", %{participation: participation, for: user}) + end + end + + def conversation_statuses( + %{assigns: %{user: user}} = conn, + %{"id" => participation_id} = params + ) do + participation = Participation.get(participation_id, preload: [:conversation]) + + if user.id == participation.user_id do + params = + params + |> Map.put("blocking_user", user) + |> Map.put("muting_user", user) + |> Map.put("user", user) + + activities = + participation.conversation.ap_id + |> ActivityPub.fetch_activities_for_context(params) + |> Enum.reverse() + + conn + |> add_link_headers(activities) + |> put_view(StatusView) + |> render("index.json", %{activities: activities, for: user, as: :activity}) + end + end + + def update_conversation( + %{assigns: %{user: user}} = conn, + %{"id" => participation_id, "recipients" => recipients} + ) do + participation = + participation_id + |> Participation.get() + + with true <- user.id == participation.user_id, + {:ok, participation} <- Participation.set_recipients(participation, recipients) do + conn + |> put_view(ConversationView) + |> render("participation.json", %{participation: participation, for: user}) + end + end + + def read_notification(%{assigns: %{user: user}} = conn, %{"id" => notification_id}) do + with {:ok, notification} <- Notification.read_one(user, notification_id) do + conn + |> put_view(NotificationView) + |> render("show.json", %{notification: notification, for: user}) + else + {:error, message} -> + conn + |> put_status(:bad_request) + |> json(%{"error" => message}) + end + end + + def read_notification(%{assigns: %{user: user}} = conn, %{"max_id" => max_id}) do + with notifications <- Notification.set_read_up_to(user, max_id) do + notifications = Enum.take(notifications, 80) + + conn + |> put_view(NotificationView) + |> render("index.json", %{notifications: notifications, for: user}) + end + end +end diff --git a/lib/pleroma/web/pleroma_api/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/emoji_api_controller.ex deleted file mode 100644 index 6beca426a..000000000 --- a/lib/pleroma/web/pleroma_api/emoji_api_controller.ex +++ /dev/null @@ -1,575 +0,0 @@ -defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do - use Pleroma.Web, :controller - - require Logger - - @emoji_dir_path Path.join( - Pleroma.Config.get!([:instance, :static_dir]), - "emoji" - ) - - @cache_seconds_per_file Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file]) - - @doc """ - Lists the packs available on the instance as JSON. - - The information is public and does not require authentification. The format is - a map of "pack directory name" to pack.json contents. - """ - def list_packs(conn, _params) do - with {:ok, results} <- File.ls(@emoji_dir_path) do - pack_infos = - results - |> Enum.filter(&has_pack_json?/1) - |> Enum.map(&load_pack/1) - # Check if all the files are in place and can be sent - |> Enum.map(&validate_pack/1) - # Transform into a map of pack-name => pack-data - |> Enum.into(%{}) - - json(conn, pack_infos) - end - end - - defp has_pack_json?(file) do - dir_path = Path.join(@emoji_dir_path, file) - # Filter to only use the pack.json packs - File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.json")) - end - - defp load_pack(pack_name) do - pack_path = Path.join(@emoji_dir_path, pack_name) - pack_file = Path.join(pack_path, "pack.json") - - {pack_name, Jason.decode!(File.read!(pack_file))} - end - - defp validate_pack({name, pack}) do - pack_path = Path.join(@emoji_dir_path, name) - - if can_download?(pack, pack_path) do - archive_for_sha = make_archive(name, pack, pack_path) - archive_sha = :crypto.hash(:sha256, archive_for_sha) |> Base.encode16() - - pack = - pack - |> put_in(["pack", "can-download"], true) - |> put_in(["pack", "download-sha256"], archive_sha) - - {name, pack} - else - {name, put_in(pack, ["pack", "can-download"], false)} - end - end - - defp can_download?(pack, pack_path) do - # If the pack is set as shared, check if it can be downloaded - # That means that when asked, the pack can be packed and sent to the remote - # Otherwise, they'd have to download it from external-src - pack["pack"]["share-files"] && - Enum.all?(pack["files"], fn {_, path} -> - File.exists?(Path.join(pack_path, path)) - end) - end - - defp create_archive_and_cache(name, pack, pack_dir, md5) do - files = - ['pack.json'] ++ - (pack["files"] |> Enum.map(fn {_, path} -> to_charlist(path) end)) - - {:ok, {_, zip_result}} = :zip.zip('#{name}.zip', files, [:memory, cwd: to_charlist(pack_dir)]) - - cache_ms = :timer.seconds(@cache_seconds_per_file * Enum.count(files)) - - Cachex.put!( - :emoji_packs_cache, - name, - # if pack.json MD5 changes, the cache is not valid anymore - %{pack_json_md5: md5, pack_data: zip_result}, - # Add a minute to cache time for every file in the pack - ttl: cache_ms - ) - - Logger.debug("Created an archive for the '#{name}' emoji pack, \ -keeping it in cache for #{div(cache_ms, 1000)}s") - - zip_result - end - - defp make_archive(name, pack, pack_dir) do - # Having a different pack.json md5 invalidates cache - pack_file_md5 = :crypto.hash(:md5, File.read!(Path.join(pack_dir, "pack.json"))) - - case Cachex.get!(:emoji_packs_cache, name) do - %{pack_file_md5: ^pack_file_md5, pack_data: zip_result} -> - Logger.debug("Using cache for the '#{name}' shared emoji pack") - zip_result - - _ -> - create_archive_and_cache(name, pack, pack_dir, pack_file_md5) - end - end - - @doc """ - An endpoint for other instances (via admin UI) or users (via browser) - to download packs that the instance shares. - """ - def download_shared(conn, %{"name" => name}) do - pack_dir = Path.join(@emoji_dir_path, name) - pack_file = Path.join(pack_dir, "pack.json") - - with {_, true} <- {:exists?, File.exists?(pack_file)}, - pack = Jason.decode!(File.read!(pack_file)), - {_, true} <- {:can_download?, can_download?(pack, pack_dir)} do - zip_result = make_archive(name, pack, pack_dir) - send_download(conn, {:binary, zip_result}, filename: "#{name}.zip") - else - {:can_download?, _} -> - conn - |> put_status(:forbidden) - |> json(%{ - error: "Pack #{name} cannot be downloaded from this instance, either pack sharing\ - was disabled for this pack or some files are missing" - }) - - {:exists?, _} -> - conn - |> put_status(:not_found) - |> json(%{error: "Pack #{name} does not exist"}) - end - end - - @doc """ - An admin endpoint to request downloading a pack named `pack_name` from the instance - `instance_address`. - - If the requested instance's admin chose to share the pack, it will be downloaded - from that instance, otherwise it will be downloaded from the fallback source, if there is one. - """ - def download_from(conn, %{"instance_address" => address, "pack_name" => name} = data) do - shareable_packs_available = - "#{address}/.well-known/nodeinfo" - |> Tesla.get!() - |> Map.get(:body) - |> Jason.decode!() - |> List.last() - |> Map.get("href") - # Get the actual nodeinfo address and fetch it - |> Tesla.get!() - |> Map.get(:body) - |> Jason.decode!() - |> get_in(["metadata", "features"]) - |> Enum.member?("shareable_emoji_packs") - - if shareable_packs_available do - full_pack = - "#{address}/api/pleroma/emoji/packs/list" - |> Tesla.get!() - |> Map.get(:body) - |> Jason.decode!() - |> Map.get(name) - - pack_info_res = - case full_pack["pack"] do - %{"share-files" => true, "can-download" => true, "download-sha256" => sha} -> - {:ok, - %{ - sha: sha, - uri: "#{address}/api/pleroma/emoji/packs/download_shared/#{name}" - }} - - %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) -> - {:ok, - %{ - sha: sha, - uri: src, - fallback: true - }} - - _ -> - {:error, - "The pack was not set as shared and there is no fallback src to download from"} - end - - with {:ok, %{sha: sha, uri: uri} = pinfo} <- pack_info_res, - %{body: emoji_archive} <- Tesla.get!(uri), - {_, true} <- {:checksum, Base.decode16!(sha) == :crypto.hash(:sha256, emoji_archive)} do - local_name = data["as"] || name - pack_dir = Path.join(@emoji_dir_path, local_name) - File.mkdir_p!(pack_dir) - - files = Enum.map(full_pack["files"], fn {_, path} -> to_charlist(path) end) - # Fallback cannot contain a pack.json file - files = if pinfo[:fallback], do: files, else: ['pack.json'] ++ files - - {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) - - # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256 - # in it to depend on itself - if pinfo[:fallback] do - pack_file_path = Path.join(pack_dir, "pack.json") - - File.write!(pack_file_path, Jason.encode!(full_pack, pretty: true)) - end - - json(conn, "ok") - else - {:error, e} -> - conn |> put_status(:internal_server_error) |> json(%{error: e}) - - {:checksum, _} -> - conn - |> put_status(:internal_server_error) - |> json(%{error: "SHA256 for the pack doesn't match the one sent by the server"}) - end - else - conn - |> put_status(:internal_server_error) - |> json(%{error: "The requested instance does not support sharing emoji packs"}) - end - end - - @doc """ - Creates an empty pack named `name` which then can be updated via the admin UI. - """ - def create(conn, %{"name" => name}) do - pack_dir = Path.join(@emoji_dir_path, name) - - if not File.exists?(pack_dir) do - File.mkdir_p!(pack_dir) - - pack_file_p = Path.join(pack_dir, "pack.json") - - File.write!( - pack_file_p, - Jason.encode!(%{pack: %{}, files: %{}}) - ) - - conn |> json("ok") - else - conn - |> put_status(:conflict) - |> json(%{error: "A pack named \"#{name}\" already exists"}) - end - end - - @doc """ - Deletes the pack `name` and all it's files. - """ - def delete(conn, %{"name" => name}) do - pack_dir = Path.join(@emoji_dir_path, name) - - case File.rm_rf(pack_dir) do - {:ok, _} -> - conn |> json("ok") - - {:error, _} -> - conn - |> put_status(:internal_server_error) - |> json(%{error: "Couldn't delete the pack #{name}"}) - end - end - - @doc """ - An endpoint to update `pack_names`'s metadata. - - `new_data` is the new metadata for the pack, that will replace the old metadata. - """ - def update_metadata(conn, %{"pack_name" => name, "new_data" => new_data}) do - pack_file_p = Path.join([@emoji_dir_path, name, "pack.json"]) - - full_pack = Jason.decode!(File.read!(pack_file_p)) - - # The new fallback-src is in the new data and it's not the same as it was in the old data - should_update_fb_sha = - not is_nil(new_data["fallback-src"]) and - new_data["fallback-src"] != full_pack["pack"]["fallback-src"] - - with {_, true} <- {:should_update?, should_update_fb_sha}, - %{body: pack_arch} <- Tesla.get!(new_data["fallback-src"]), - {:ok, flist} <- :zip.unzip(pack_arch, [:memory]), - {_, true} <- {:has_all_files?, has_all_files?(full_pack, flist)} do - fallback_sha = :crypto.hash(:sha256, pack_arch) |> Base.encode16() - - new_data = Map.put(new_data, "fallback-src-sha256", fallback_sha) - update_metadata_and_send(conn, full_pack, new_data, pack_file_p) - else - {:should_update?, _} -> - update_metadata_and_send(conn, full_pack, new_data, pack_file_p) - - {:has_all_files?, _} -> - conn - |> put_status(:bad_request) - |> json(%{error: "The fallback archive does not have all files specified in pack.json"}) - end - end - - # Check if all files from the pack.json are in the archive - defp has_all_files?(%{"files" => files}, flist) do - Enum.all?(files, fn {_, from_manifest} -> - Enum.find(flist, fn {from_archive, _} -> - to_string(from_archive) == from_manifest - end) - end) - end - - defp update_metadata_and_send(conn, full_pack, new_data, pack_file_p) do - full_pack = Map.put(full_pack, "pack", new_data) - File.write!(pack_file_p, Jason.encode!(full_pack, pretty: true)) - - # Send new data back with fallback sha filled - json(conn, new_data) - end - - defp get_filename(%{"filename" => filename}), do: filename - - defp get_filename(%{"file" => file}) do - case file do - %Plug.Upload{filename: filename} -> filename - url when is_binary(url) -> Path.basename(url) - end - end - - defp empty?(str), do: String.trim(str) == "" - - defp update_file_and_send(conn, updated_full_pack, pack_file_p) do - # Write the emoji pack file - File.write!(pack_file_p, Jason.encode!(updated_full_pack, pretty: true)) - - # Return the modified file list - json(conn, updated_full_pack["files"]) - end - - @doc """ - Updates a file in a pack. - - Updating can mean three things: - - - `add` adds an emoji named `shortcode` to the pack `pack_name`, - that means that the emoji file needs to be uploaded with the request - (thus requiring it to be a multipart request) and be named `file`. - There can also be an optional `filename` that will be the new emoji file name - (if it's not there, the name will be taken from the uploaded file). - - `update` changes emoji shortcode (from `shortcode` to `new_shortcode` or moves the file - (from the current filename to `new_filename`) - - `remove` removes the emoji named `shortcode` and it's associated file - """ - - # Add - def update_file( - conn, - %{"pack_name" => pack_name, "action" => "add", "shortcode" => shortcode} = params - ) do - pack_dir = Path.join(@emoji_dir_path, pack_name) - pack_file_p = Path.join(pack_dir, "pack.json") - - full_pack = Jason.decode!(File.read!(pack_file_p)) - - with {_, false} <- {:has_shortcode, Map.has_key?(full_pack["files"], shortcode)}, - filename <- get_filename(params), - false <- empty?(shortcode), - false <- empty?(filename) do - file_path = Path.join(pack_dir, filename) - - # If the name contains directories, create them - if String.contains?(file_path, "/") do - File.mkdir_p!(Path.dirname(file_path)) - end - - case params["file"] do - %Plug.Upload{path: upload_path} -> - # Copy the uploaded file from the temporary directory - File.copy!(upload_path, file_path) - - url when is_binary(url) -> - # Download and write the file - file_contents = Tesla.get!(url).body - File.write!(file_path, file_contents) - end - - updated_full_pack = put_in(full_pack, ["files", shortcode], filename) - update_file_and_send(conn, updated_full_pack, pack_file_p) - else - {:has_shortcode, _} -> - conn - |> put_status(:conflict) - |> json(%{error: "An emoji with the \"#{shortcode}\" shortcode already exists"}) - - true -> - conn - |> put_status(:bad_request) - |> json(%{error: "shortcode or filename cannot be empty"}) - end - end - - # Remove - def update_file(conn, %{ - "pack_name" => pack_name, - "action" => "remove", - "shortcode" => shortcode - }) do - pack_dir = Path.join(@emoji_dir_path, pack_name) - pack_file_p = Path.join(pack_dir, "pack.json") - - full_pack = Jason.decode!(File.read!(pack_file_p)) - - if Map.has_key?(full_pack["files"], shortcode) do - {emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) - - emoji_file_path = Path.join(pack_dir, emoji_file_path) - - # Delete the emoji file - File.rm!(emoji_file_path) - - # If the old directory has no more files, remove it - if String.contains?(emoji_file_path, "/") do - dir = Path.dirname(emoji_file_path) - - if Enum.empty?(File.ls!(dir)) do - File.rmdir!(dir) - end - end - - update_file_and_send(conn, updated_full_pack, pack_file_p) - else - conn - |> put_status(:bad_request) - |> json(%{error: "Emoji \"#{shortcode}\" does not exist"}) - end - end - - # Update - def update_file( - conn, - %{"pack_name" => pack_name, "action" => "update", "shortcode" => shortcode} = params - ) do - pack_dir = Path.join(@emoji_dir_path, pack_name) - pack_file_p = Path.join(pack_dir, "pack.json") - - full_pack = Jason.decode!(File.read!(pack_file_p)) - - with {_, true} <- {:has_shortcode, Map.has_key?(full_pack["files"], shortcode)}, - %{"new_shortcode" => new_shortcode, "new_filename" => new_filename} <- params, - false <- empty?(new_shortcode), - false <- empty?(new_filename) do - # First, remove the old shortcode, saving the old path - {old_emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) - old_emoji_file_path = Path.join(pack_dir, old_emoji_file_path) - new_emoji_file_path = Path.join(pack_dir, new_filename) - - # If the name contains directories, create them - if String.contains?(new_emoji_file_path, "/") do - File.mkdir_p!(Path.dirname(new_emoji_file_path)) - end - - # Move/Rename the old filename to a new filename - # These are probably on the same filesystem, so just rename should work - :ok = File.rename(old_emoji_file_path, new_emoji_file_path) - - # If the old directory has no more files, remove it - if String.contains?(old_emoji_file_path, "/") do - dir = Path.dirname(old_emoji_file_path) - - if Enum.empty?(File.ls!(dir)) do - File.rmdir!(dir) - end - end - - # Then, put in the new shortcode with the new path - updated_full_pack = put_in(updated_full_pack, ["files", new_shortcode], new_filename) - update_file_and_send(conn, updated_full_pack, pack_file_p) - else - {:has_shortcode, _} -> - conn - |> put_status(:bad_request) - |> json(%{error: "Emoji \"#{shortcode}\" does not exist"}) - - true -> - conn - |> put_status(:bad_request) - |> json(%{error: "new_shortcode or new_filename cannot be empty"}) - - _ -> - conn - |> put_status(:bad_request) - |> json(%{error: "new_shortcode or new_file were not specified"}) - end - end - - def update_file(conn, %{"action" => action}) do - conn - |> put_status(:bad_request) - |> json(%{error: "Unknown action: #{action}"}) - end - - @doc """ - Imports emoji from the filesystem. - - Importing means checking all the directories in the - `$instance_static/emoji/` for directories which do not have - `pack.json`. If one has an emoji.txt file, that file will be used - to create a `pack.json` file with it's contents. If the directory has - neither, all the files with specific configured extenstions will be - assumed to be emojis and stored in the new `pack.json` file. - """ - def import_from_fs(conn, _params) do - with {:ok, results} <- File.ls(@emoji_dir_path) do - imported_pack_names = - results - |> Enum.filter(fn file -> - dir_path = Path.join(@emoji_dir_path, file) - # Find the directories that do NOT have pack.json - File.dir?(dir_path) and not File.exists?(Path.join(dir_path, "pack.json")) - end) - |> Enum.map(&write_pack_json_contents/1) - - json(conn, imported_pack_names) - else - {:error, _} -> - conn - |> put_status(:internal_server_error) - |> json(%{error: "Error accessing emoji pack directory"}) - end - end - - defp write_pack_json_contents(dir) do - dir_path = Path.join(@emoji_dir_path, dir) - emoji_txt_path = Path.join(dir_path, "emoji.txt") - - files_for_pack = files_for_pack(emoji_txt_path, dir_path) - pack_json_contents = Jason.encode!(%{pack: %{}, files: files_for_pack}) - - File.write!(Path.join(dir_path, "pack.json"), pack_json_contents) - - dir - end - - defp files_for_pack(emoji_txt_path, dir_path) do - if File.exists?(emoji_txt_path) do - # There's an emoji.txt file, it's likely from a pack installed by the pack manager. - # Make a pack.json file from the contents of that emoji.txt fileh - - # FIXME: Copy-pasted from Pleroma.Emoji/load_from_file_stream/2 - - # Create a map of shortcodes to filenames from emoji.txt - File.read!(emoji_txt_path) - |> String.split("\n") - |> Enum.map(&String.trim/1) - |> Enum.map(fn line -> - case String.split(line, ~r/,\s*/) do - # This matches both strings with and without tags - # and we don't care about tags here - [name, file | _] -> {name, file} - _ -> nil - end - end) - |> Enum.filter(fn x -> not is_nil(x) end) - |> Enum.into(%{}) - else - # If there's no emoji.txt, assume all files - # that are of certain extensions from the config are emojis and import them all - pack_extensions = Pleroma.Config.get!([:emoji, :pack_extensions]) - Pleroma.Emoji.make_shortcode_to_file_map(dir_path, pack_extensions) - end - end -end diff --git a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex deleted file mode 100644 index d17ccf84d..000000000 --- a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex +++ /dev/null @@ -1,89 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do - use Pleroma.Web, :controller - - import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2] - - alias Pleroma.Conversation.Participation - alias Pleroma.Notification - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.MastodonAPI.ConversationView - alias Pleroma.Web.MastodonAPI.NotificationView - alias Pleroma.Web.MastodonAPI.StatusView - - def conversation(%{assigns: %{user: user}} = conn, %{"id" => participation_id}) do - with %Participation{} = participation <- Participation.get(participation_id), - true <- user.id == participation.user_id do - conn - |> put_view(ConversationView) - |> render("participation.json", %{participation: participation, for: user}) - end - end - - def conversation_statuses( - %{assigns: %{user: user}} = conn, - %{"id" => participation_id} = params - ) do - participation = Participation.get(participation_id, preload: [:conversation]) - - if user.id == participation.user_id do - params = - params - |> Map.put("blocking_user", user) - |> Map.put("muting_user", user) - |> Map.put("user", user) - - activities = - participation.conversation.ap_id - |> ActivityPub.fetch_activities_for_context(params) - |> Enum.reverse() - - conn - |> add_link_headers(activities) - |> put_view(StatusView) - |> render("index.json", %{activities: activities, for: user, as: :activity}) - end - end - - def update_conversation( - %{assigns: %{user: user}} = conn, - %{"id" => participation_id, "recipients" => recipients} - ) do - participation = - participation_id - |> Participation.get() - - with true <- user.id == participation.user_id, - {:ok, participation} <- Participation.set_recipients(participation, recipients) do - conn - |> put_view(ConversationView) - |> render("participation.json", %{participation: participation, for: user}) - end - end - - def read_notification(%{assigns: %{user: user}} = conn, %{"id" => notification_id}) do - with {:ok, notification} <- Notification.read_one(user, notification_id) do - conn - |> put_view(NotificationView) - |> render("show.json", %{notification: notification, for: user}) - else - {:error, message} -> - conn - |> put_status(:bad_request) - |> json(%{"error" => message}) - end - end - - def read_notification(%{assigns: %{user: user}} = conn, %{"max_id" => max_id}) do - with notifications <- Notification.set_read_up_to(user, max_id) do - notifications = Enum.take(notifications, 80) - - conn - |> put_view(NotificationView) - |> render("index.json", %{notifications: notifications, for: user}) - end - end -end -- cgit v1.2.3