Skip to content

Commit

Permalink
feat: search photo's caption (#21)
Browse files Browse the repository at this point in the history 
* keep silence when upload and no same photos found

style: 💄 clean code

* refactor: 💡 clean code

* fix: 🐛 belongs_to_id
  • Loading branch information
@ThaddeusJiang
ThaddeusJiang authored Oct 23, 2024
1 parent b3ac13d commit 45cd19a
Show file tree
Hide file tree
Showing 2 changed files with 104 additions and 82 deletions.
169 changes: 99 additions & 70 deletions lib/save_it/bot.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,13 @@ defmodule SaveIt.Bot do
setup_commands: true

command("start")
command("search", description: "Search similar photos by photo")
command("similar", description: "Find similar photos")
command("search", description: "Search photos")
command("about", description: "About the bot")

command("login", description: "Login")
command("code", description: "Get code for login")
command("folder", description: "Update Google Drive folder ID")
command("about", description: "About the bot")

middleware(ExGram.Middleware.IgnoreUsername)

Expand Down Expand Up @@ -113,23 +115,109 @@ defmodule SaveIt.Bot do
end
end

def handle({:command, :search, %{chat: chat, photo: nil, text: q}}, _context) do
photos = TypesensePhoto.search_photos!(q: q, belongs_to_id: chat.id)
def handle({:command, :search, %{chat: chat, text: text}}, _context)
when is_binary(text) do
q = String.trim(text)

case q do
"" ->
send_message(chat.id, "What do you want to search? animal, food, etc.")

_ ->
photos = TypesensePhoto.search_photos!(q, belongs_to_id: chat.id)

answer_photos(chat.id, photos)
end
end

answer_photos(chat.id, photos)
def handle({:command, :similar, %{chat: chat, photo: nil}}, _context) do
send_message(chat.id, "Upload a photo to find similar photos.")
end

def handle({:command, :search, %{chat: chat, photo: nil}}, _context) do
send_message(chat.id, "Please send me a photo to search.")
# dev-notes: never reach here, it will be handled by handle({:message, %{chat: chat, caption: nil, photo: photos}}, ctx)
# def handle({:command, :similar, %{chat: _chat, photo: photo}}, _context) do
# end

# caption: nil -> find same photos
def handle({:message, %{chat: chat, caption: nil, photo: photos}}, ctx) do
photo = List.last(photos)

file = ExGram.get_file!(photo.file_id)
photo_file_content = Telegram.download_file_content!(file.file_path)

bot_id = ctx.bot_info.id
chat_id = chat.id

typesense_photo =
TypesensePhoto.create_photo!(%{
image: Base.encode64(photo_file_content),
caption: "",
url: photo_url(bot_id, file.file_id),
belongs_to_id: chat_id
})

photos =
TypesensePhoto.search_similar_photos!(
typesense_photo["id"],
distance_threshold: 0.1,
belongs_to_id: chat_id
)

case photos do
[] -> nil
_ -> answer_photos(chat.id, photos)
end
end

# caption: contains /similar or /search -> search similar photos; otherwise, find same photos
def handle({:message, %{chat: chat, caption: caption, photo: photos}}, ctx) do
photo = List.last(photos)

search_similar_photos_based_on_caption(photo, caption,
chat_id: chat.id,
bot_id: ctx.bot_info.id
)
file = ExGram.get_file!(photo.file_id)
photo_file_content = Telegram.download_file_content!(file.file_path)

bot_id = ctx.bot_info.id
chat_id = chat.id

caption =
if String.contains?(caption, ["/similar", "/search"]) do
""
else
caption
end

typesense_photo =
TypesensePhoto.create_photo!(%{
image: Base.encode64(photo_file_content),
caption: caption,
url: photo_url(bot_id, file.file_id),
belongs_to_id: chat_id
})

case caption do
"" ->
photos =
TypesensePhoto.search_similar_photos!(
typesense_photo["id"],
distance_threshold: 0.4,
belongs_to_id: chat_id
)

answer_photos(chat.id, photos)

_ ->
photos =
TypesensePhoto.search_similar_photos!(
typesense_photo["id"],
distance_threshold: 0.1,
belongs_to_id: chat_id
)

case photos do
[] -> nil
_ -> answer_photos(chat.id, photos)
end
end
end

def handle({:text, text, %{chat: chat, message_id: message_id}}, _context) do
Expand Down Expand Up @@ -246,32 +334,6 @@ defmodule SaveIt.Bot do
{:ok, nil}
end

defp search_similar_photos(photo, opts) do
file = ExGram.get_file!(photo.file_id)

photo_file_content = Telegram.download_file_content!(file.file_path)

bot_id = Keyword.get(opts, :bot_id)
chat_id = Keyword.get(opts, :chat_id)
distance_threshold = Keyword.get(opts, :distance_threshold, 0.4)

typesense_photo =
TypesensePhoto.create_photo!(%{
image: Base.encode64(photo_file_content),
caption: Map.get(photo, "caption", ""),
url: photo_url(bot_id, file.file_id),
belongs_to_id: chat_id
})

if typesense_photo != nil do
TypesensePhoto.search_photos!(
typesense_photo["id"],
distance_threshold: distance_threshold,
belongs_to_id: chat_id
)
end
end

defp pick_file_id_from_photo_url(photo_url) do
captures =
Regex.named_captures(~r"/files/(?<bot_id>\d+)/(?<file_id>.+)", photo_url)
Expand Down Expand Up @@ -342,18 +404,6 @@ defmodule SaveIt.Bot do
Enum.each(message_ids, fn message_id -> delete_message(chat_id, message_id) end)
end

# defp bot_send_media_group(chat_id, files) do
# media =
# Enum.map(files, fn {file_name, file_content} ->
# %ExGram.Model.InputMediaDocument{
# type: "document",
# media: file_content
# }
# end)

# ExGram.send_media_group(chat_id, media)
# end

defp bot_send_files(chat_id, files) do
Enum.each(files, fn {file_name, file_content} ->
bot_send_file(chat_id, file_name, {:file_content, file_content, file_name})
Expand Down Expand Up @@ -445,25 +495,4 @@ defmodule SaveIt.Bot do
encoded_file_id = URI.encode(file_id)
"#{proxy_url}/#{encoded_bot_id}/#{encoded_file_id}"
end

defp search_similar_photos_based_on_caption(photo, caption, opts) do
bot_id = Keyword.get(opts, :bot_id)
chat_id = Keyword.get(opts, :chat_id)

distance_threshold =
if caption && String.contains?(caption, "/search") do
0.4
else
0.1
end

similar_photos =
search_similar_photos(photo,
distance_threshold: distance_threshold,
bot_id: bot_id,
chat_id: chat_id
)

answer_photos(chat_id, similar_photos)
end
end
17 changes: 5 additions & 12 deletions lib/save_it/typesense_photo.ex
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
defmodule SaveIt.TypesensePhoto do
require Logger
alias SmallSdk.Typesense

def create_photo!(
Expand All @@ -25,12 +26,14 @@ defmodule SaveIt.TypesensePhoto do
Typesense.get_document("photos", photo_id)
end

def search_photos!(q: q, belongs_to_id: belongs_to_id) do
def search_photos!(q, opts) do
belongs_to_id = Keyword.get(opts, :belongs_to_id)

req_body = %{
"searches" => [
%{
"query_by" => "image_embedding,caption",
"q" => q,
"query_by" => "image_embedding",
"collection" => "photos",
"prefix" => false,
"vector_query" => "image_embedding:([], k: 5, distance_threshold: 0.75)",
Expand All @@ -46,16 +49,6 @@ defmodule SaveIt.TypesensePhoto do
res.body["results"] |> typesense_results_to_documents()
end

def search_photos!(id, opts \\ []) do
belongs_to_id = Keyword.get(opts, :belongs_to_id)
distance_threshold = Keyword.get(opts, :distance_threshold, 0.4)

search_similar_photos!(id,
distance_threshold: distance_threshold,
belongs_to_id: belongs_to_id
)
end

def search_similar_photos!(photo_id, opts \\ []) when is_binary(photo_id) do
belongs_to_id = Keyword.get(opts, :belongs_to_id)
distance_threshold = Keyword.get(opts, :distance_threshold, 0.4)
Expand Down

0 comments on commit 45cd19a

Please sign in to comment.