Merge branch 'dedupe-sharding' into 'develop'

Pleroma.Upload.Filter.Dedupe: sharding directory structure

See merge request pleroma/pleroma!4292

3 files changed, 15 insertions, 7 deletions

diff --git a/test/pleroma/object_test.exs b/test/pleroma/object_test.exs
index b3c528e32..ed5c2b6c8 100644
--- a/test/pleroma/object_test.exs
+++ b/test/pleroma/object_test.exs
@@ -174,8 +174,9 @@ defmodule Pleroma.ObjectTest do
 
       filename = Path.basename(href)
 
-      assert {:ok, files} = File.ls(uploads_dir)
-      assert filename in files
+      expected_path = Path.join([uploads_dir, Pleroma.Upload.Filter.Dedupe.shard_path(filename)])
+
+      assert File.exists?(expected_path)
 
       Object.delete(note)
 
@@ -183,8 +184,7 @@ defmodule Pleroma.ObjectTest do
 
       assert Object.get_by_id(note.id).data["deleted"]
       assert Object.get_by_id(attachment.id) == nil
-      assert {:ok, files} = File.ls(uploads_dir)
-      refute filename in files
+      refute File.exists?(expected_path)
     end
 
     test "with objects that have legacy data.url attribute" do
diff --git a/test/pleroma/upload/filter/dedupe_test.exs b/test/pleroma/upload/filter/dedupe_test.exs
index 29c181509..4dc28b998 100644
--- a/test/pleroma/upload/filter/dedupe_test.exs
+++ b/test/pleroma/upload/filter/dedupe_test.exs
@@ -10,6 +10,10 @@ defmodule Pleroma.Upload.Filter.DedupeTest do
 
   @shasum "e30397b58d226d6583ab5b8b3c5defb0c682bda5c31ef07a9f57c1c4986e3781"
 
+  test "generates a shard path for a shasum" do
+    assert "e3/03/97/" <> _path = Dedupe.shard_path(@shasum)
+  end
+
   test "adds shasum" do
     File.cp!(
       "test/fixtures/image.jpg",
@@ -23,10 +27,12 @@ defmodule Pleroma.Upload.Filter.DedupeTest do
       tempfile: Path.absname("test/fixtures/image_tmp.jpg")
     }
 
+    expected_path = Dedupe.shard_path(@shasum <> ".jpg")
+
     assert {
              :ok,
              :filtered,
-             %Pleroma.Upload{id: @shasum, path: @shasum <> ".jpg"}
+             %Pleroma.Upload{id: @shasum, path: ^expected_path}
            } = Dedupe.filter(upload)
   end
 end
diff --git a/test/pleroma/upload_test.exs b/test/pleroma/upload_test.exs
index facb634c3..5fd62fa43 100644
--- a/test/pleroma/upload_test.exs
+++ b/test/pleroma/upload_test.exs
@@ -149,6 +149,9 @@ defmodule Pleroma.UploadTest do
 
     test "copies the file to the configured folder with deduping" do
       File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
+      expected_filename = "e30397b58d226d6583ab5b8b3c5defb0c682bda5c31ef07a9f57c1c4986e3781.jpg"
+
+      expected_path = Pleroma.Upload.Filter.Dedupe.shard_path(expected_filename)
 
       file = %Plug.Upload{
         content_type: "image/jpeg",
@@ -159,8 +162,7 @@ defmodule Pleroma.UploadTest do
       {:ok, data} = Upload.store(file, filters: [Pleroma.Upload.Filter.Dedupe])
 
       assert List.first(data["url"])["href"] ==
-               Pleroma.Upload.base_url() <>
-                 "e30397b58d226d6583ab5b8b3c5defb0c682bda5c31ef07a9f57c1c4986e3781.jpg"
+               Path.join([Pleroma.Upload.base_url(), expected_path])
     end
 
     test "copies the file to the configured folder without deduping" do